libexpr/static-analyzer: init

Change-Id: I606718fbe33db1f6c00ca34f45c6553ce468398c
Signed-off-by: Raito Bezarius <raito@lix.systems>

.clang-tidy

@@ -20,6 +20,8 @@ Checks:
   - -bugprone-multi-level-implicit-pointer-conversion
     # we don't compile out our asserts
   - -bugprone-assert-side-effect
+    # FIXME(jade): figure out if this warning is any good
+  - -bugprone-exception-escape
     # all thrown exceptions must derive from std::exception
   - hicpp-exception-baseclass
     # capturing async lambdas are dangerous

.editorconfig

@@ -33,3 +33,7 @@ max_line_length = 0
 [meson.build]
 indent_style = space
 indent_size = 2
+
+[*.json]
+indent_style = space
+indent_size = 4

.gitignore

@@ -39,3 +39,5 @@ buildtime.bin
 
 # Python compiled files from the code generators and test suite
 *.pyc
+
+**/.idea

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "countme"
@@ -47,9 +47,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "rnix"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb35cedbeb70e0ccabef2a31bcff0aebd114f19566086300b8f42c725fc2cb5f"
+checksum = "6f15e00b0ab43abd70d50b6f8cd021290028f9b7fdd7cdfa6c35997173bc1ba9"
 dependencies = [
  "rowan",
 ]

OWNERS

@@ -0,0 +1,26 @@
+# Docs
+per-file README.md=*
+per-file CONTRIBUTING.md=*
+
+# DevX
+per-file justfile=*
+per-file .envrc=*
+per-file .gitignore=*
+per-file .github=*
+per-file .mailmap=*
+
+# Build
+per-file meson.build=*
+per-file meson.options=*
+per-file flake.nix=*
+per-file flake.lock=*
+per-file *.nix=*
+per-file Cargo.lock=*
+per-file Cargo.toml=*
+per-file version.json=*
+
+# Code style
+per-file .clang-tidy=*
+per-file .clang-format=*
+per-file .editorconfig=*
+per-file treefmt.toml=*

bench/OWNERS

@@ -0,0 +1 @@
+*

bench/bench.py

@@ -7,15 +7,44 @@ import os
 import json
 import tempfile
 import platform
+import shlex
+import textwrap
+import dataclasses
 
-flake_args = ["--extra-experimental-features","'nix-command flakes'"]
-# hyperfine has its own variable substitution, so we use that and pass build="{BUILD}" here.
-# perf doesn't have variable substitution, so we call these with build being the actual build directory.
+flake_args = ["--extra-experimental-features", "nix-command flakes"]
 cases = {
-    "search": lambda build: [f"{build}/bin/nix", *flake_args, "search", "--no-eval-cache", "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870", "hello"],
-    "rebuild": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
-    "rebuild_lh": lambda build: ["GC_INITIAL_HEAP_SIZE=10g", f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
-    "parse": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "-f", "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"],
+    "search": lambda build: [
+        f"{build}/bin/nix",
+        *flake_args,
+        "search",
+        "--no-eval-cache",
+        "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870",
+        "hello",
+    ],
+    "rebuild": lambda build: [
+        f"{build}/bin/nix",
+        *flake_args,
+        "eval",
+        "--raw",
+        "--impure",
+        "--expr",
+        textwrap.dedent("""
+            (import <nixpkgs/nixos> {
+                configuration = ./bench/nixpkgs/nixos/modules/installer/cd-dvd/installation-cd-graphical-calamares-plasma6.nix;
+            }).config.system.build.toplevel
+        """).replace("\n", " "),
+    ],
+    "rebuild_lh": lambda build: [
+        "GC_INITIAL_HEAP_SIZE=10g",
+        *cases['rebuild'](build),
+    ],
+    "parse": lambda build: [
+        f"{build}/bin/nix",
+        *flake_args,
+        "eval",
+        "-f",
+        "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix",
+    ],
 }
 
 arg_parser = argparse.ArgumentParser()
@@ -24,46 +53,81 @@ arg_parser = argparse.ArgumentParser()
 # mode, we would have to combine the JSON ourselves to support that, which
 # would probably be better done by writing a benchmarking script in
 # not-bash.
-arg_parser.add_argument('builds', nargs='+', help="At least two build directories to compare, containing bin/nix")
-arg_parser.add_argument('--cases', type=str, help="A comma-separated list of cases you want to run. Defaults to running all")
-available_modes = [ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [] # perf doesn't run on Darwin
-arg_parser.add_argument('--mode', choices=available_modes, default="walltime")
+arg_parser.add_argument(
+    'builds',
+    nargs='+',
+    help="At least two build directories to compare, containing bin/nix",
+)
+arg_parser.add_argument(
+    '--cases',
+    type=str,
+    help="A comma-separated list of cases you want to run. Defaults to running all",
+)
+arg_parser.add_argument(
+    '--mode',
+    nargs='+',
+    choices=[ "walltime", "memory" ] + [ "icount" ] if platform.system() == 'Linux' else [], # perf doesn't run on Darwin
+    default=[ "walltime" ],
+)
+arg_parser.add_argument(
+    '--daemon',
+    action='store_true',
+    help='Run a temporary daemon for the benchmark instead of using a local store directly',
+)
 args = arg_parser.parse_args()
-if len(args.builds) < 2:
-    raise ValueError("need at least two build directories to compare")
+if len(args.builds) < 1:
+    raise ValueError("need at least one build directory to benchmark")
 
 benchmarks: list[str] = []
 if args.cases is None:
     benchmarks = list(cases.keys())
 else:
     for case in args.cases.split(","):
-        if case not in cases: raise ValueError(f"no such case: {case}")
+        if case not in cases:
+            raise ValueError(f"no such case: {case}")
         benchmarks.append(case)
 
+def make_full_command(build, case):
+    cmd = " ".join(map(shlex.quote, cases[case](build)))
+    if args.daemon:
+        return " ".join([
+            f"{build}/bin/nix --extra-experimental-features nix-command daemon &",
+            "trap 'kill %1' EXIT;",
+            f"NIX_REMOTE=daemon {cmd}",
+        ])
+    else:
+        return cmd
+
 def bench_walltime(env):
-    hyperfine_args = ["--parameter-list", "BUILD", ','.join(args.builds), "--warmup", "2", "--runs", "10"]
     for case in benchmarks:
-        case_command = cases[case]("{BUILD}") # see the comment on cases
-        subprocess.run([
-            "taskset", "-c", "2,3",
-            "chrt", "-f","50",
-            "hyperfine", *hyperfine_args, "--export-json", f"bench/bench-{case}.json", "--export-markdown", f"bench/bench-{case}.md", "--", " ".join(case_command)
-        ], env=env, check=True)
+        for build in args.builds:
+            subprocess.run([
+                "taskset", "-c", "2,3",
+                "chrt", "-f","50",
+                *[
+                    "hyperfine", "--warmup", "2", "--runs", "10",
+                    "--export-json", f"bench/bench-{case}-{build}.json",
+                    "--export-markdown", f"bench/bench-{case}-{build}.md",
+                    "--", make_full_command(build, case),
+                ],
+            ], env=env, check=True)
 
     print("Benchmarks summary\n---\n")
     for case in benchmarks:
-        fd = open(f"bench/bench-{case}.json")
-        result_json = json.load(fd)
-        fd.close()
-        for result in result_json["results"]:
+        results = []
+        for build in args.builds:
+            with open(f"bench/bench-{case}-{build}.json") as fd:
+                results.append(json.load(fd)["results"][0])
+        for result in results:
             print(result["command"])
             print("-" * min(80,len(result["command"])))
-            attr_rounded = lambda attr: f"{result[attr]:.3f}"
+            def attr_rounded(attr):
+                return f"{result[attr]:.3f}"
             print("  mean:    ", attr_rounded("mean"), "±", attr_rounded("stddev"))
             print("            user:",  attr_rounded("user"), "| system", attr_rounded("system"))
             print("  median:  ", attr_rounded("median"))
             print("  range:   ", attr_rounded("min") + "s.." + attr_rounded("max")+"s")
-            print("  relative:", f"{result["mean"]/result_json["results"][0]["mean"]:.3f}")
+            print("  relative:", f"{result["mean"]/results[0]["mean"]:.3f}")
             print("\n")
 
 
@@ -71,12 +135,14 @@ def bench_icount(env):
     perf_results_for: dict[str, list[tuple[str, float]]] = {}
     for case in benchmarks:
         for build in args.builds:
-            case_command = cases[case](build)
             # the perf stat -j output (incorrectly) localizes numbers, which will trip up the json parser.
             env["LC_ALL"]="C"
+            case_command = make_full_command(build, case)
             commandline = [
-                "perf", "stat", "-o", f"bench/perf-{case}.json", "-j", "sh", "-c", " ".join(case_command)
+                "perf", "stat", "-o", f"bench/perf-{case}.json", "-j",
+                "sh", "-c", case_command,
             ]
+            print("running", case_command)
             subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL) # warmup run
             subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL)
             perf_fd = open(f"bench/perf-{case}.json")
@@ -84,8 +150,9 @@ def bench_icount(env):
             perf_fd.close()
 
             instr = next(x for x in perf_data if x["event"] in ["instructions", "instructions:u"]) # an implementation of a find_first iterator
-            if case not in perf_results_for: perf_results_for[case] = []
-            perf_results_for[case].append((" ".join(case_command), float(instr["counter-value"])))
+            if case not in perf_results_for:
+                perf_results_for[case] = []
+            perf_results_for[case].append((case_command, float(instr["counter-value"])))
 
     print("Benchmarks summary\n---\n")
     for (case, entries) in perf_results_for.items():
@@ -97,6 +164,54 @@ def bench_icount(env):
             print("  relative instructions:", int(instr)/perf_results_for[case][0][1])
             print("\n")
 
+@dataclasses.dataclass
+class MemoryStatistics:
+    envBytes: int
+    listBytes: int
+    setBytes: int
+    valueBytes: int
+    heapBytes: int
+    heapSize: int
+
+def bench_memory(env):
+    path = "bench/bench-memory.json"
+    env = env | {
+        'NIX_SHOW_STATS': '1',
+        'NIX_SHOW_STATS_PATH': path,
+    }
+    results: dict[str, list[tuple[str, MemoryStatistics]]] = {}
+    for case in benchmarks:
+        for build in args.builds:
+            case_command = make_full_command(build, case)
+            commandline = [ "sh", "-c", case_command ]
+            print("running", case_command)
+            subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            with open(path) as fd:
+                stats = json.load(fd)
+            results.setdefault(case, []).append((case_command, MemoryStatistics(
+                envBytes=stats['envs']['bytes'],
+                listBytes=stats['list']['bytes'],
+                setBytes=stats['sets']['bytes'],
+                valueBytes=stats['values']['bytes'],
+                heapSize=stats['gc']['heapSize'],
+                heapBytes=stats['gc']['totalBytes'],
+            )))
+
+    print("Benchmarks summary\n---\n")
+    for (case, entries) in results.items():
+        for cmd, stats in entries:
+            print(cmd)
+            print("-" * min(80, len(cmd)))
+            print(f"  env bytes:    {stats.envBytes  :15d}   |  {(stats.envBytes / entries[0][1].envBytes)    :.3f}x")
+            print(f"  list bytes:   {stats.listBytes :15d}   |  {(stats.listBytes / entries[0][1].listBytes)  :.3f}x")
+            print(f"  set bytes:    {stats.setBytes  :15d}   |  {(stats.setBytes / entries[0][1].setBytes)    :.3f}x")
+            if not entries[0][1].valueBytes:
+                print(f"  value bytes:  {0:15d}")
+            else:
+                print(f"  value bytes:  {stats.valueBytes:15d}   |  {(stats.valueBytes / entries[0][1].valueBytes):.3f}x")
+            print(f"  heap alloc'd: {stats.heapBytes :15d}   |  {(stats.heapBytes / entries[0][1].heapBytes)  :.3f}x")
+            print(f"  heap size:    {stats.heapSize  :15d}   |  {(stats.heapSize / entries[0][1].heapSize)    :.3f}x")
+            print("\n")
 
 with tempfile.TemporaryDirectory() as tmp_dir:
     subprocess.run([
@@ -108,9 +223,15 @@ with tempfile.TemporaryDirectory() as tmp_dir:
     subenv = os.environ.copy()
     subenv["NIX_CONF_DIR"] = "/var/empty"
     subenv["NIX_REMOTE"] = tmp_dir
-    subenv["NIX_PATH"] = "nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
+    subenv["NIX_PATH"] = ":".join([
+        "nixpkgs=bench/nixpkgs",
+    ])
+    subenv["NIX_DAEMON_SOCKET_PATH"] = f"{tmp_dir}/daemon"
 
-    if args.mode == "walltime":
-        bench_walltime(subenv)
-    else:
-        bench_icount(subenv)
+    for mode in args.mode:
+        if mode == "walltime":
+            bench_walltime(subenv)
+        elif mode == "memory":
+            bench_memory(subenv)
+        else:
+            bench_icount(subenv)

bench/configuration.nix

@@ -1,314 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-
-{
-  boot = {
-    initrd = {
-      availableKernelModules = [
-        "xhci_pci"
-        "ahci"
-      ];
-      kernelModules = [ "dm-snapshot" ];
-      luks.devices = {
-        croot = {
-          device = "/dev/sdb";
-          allowDiscards = true;
-        };
-      };
-    };
-    kernelModules = [ "kvm-intel" ];
-    kernelPackages = pkgs.linuxPackages_latest;
-
-    loader = {
-      systemd-boot.enable = true;
-      efi.canTouchEfiVariables = true;
-    };
-  };
-
-  hardware = {
-    enableRedistributableFirmware = true;
-    cpu.intel.updateMicrocode = true;
-    graphics.enable32Bit = true;
-    graphics.extraPackages = with pkgs; [
-      vaapiIntel
-      intel-media-driver
-      intel-compute-runtime
-    ];
-  };
-
-  fileSystems = {
-    "/" = {
-      device = "/dev/sda2";
-      fsType = "xfs";
-      options = [ "noatime" ];
-    };
-
-    "/boot" = {
-      device = "/dev/sda1";
-      fsType = "vfat";
-    };
-
-    "/nas" = {
-      device = "nas:/";
-      fsType = "nfs4";
-      options = [
-        "ro"
-        "x-systemd.automount"
-      ];
-    };
-  };
-  swapDevices = [ { device = "/dev/swap"; } ];
-
-  networking = {
-    useDHCP = false;
-    hostName = "host";
-    wireless = {
-      enable = true;
-      interfaces = [ "eth1" ];
-    };
-    interfaces = {
-      eth0.useDHCP = true;
-      eth1.useDHCP = true;
-    };
-    wg-quick.interfaces = {
-      wg0 = {
-        address = [ "2001:db8::1" ];
-        privateKeyFile = "/etc/secrets/wg0.key";
-        peers = [
-          {
-            publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
-            endpoint = "[2001:db8::2]:61021";
-            allowedIPs = [ "2001::db8:1::/64" ];
-          }
-        ];
-      };
-    };
-
-    firewall.allowedUDPPorts = [ 4567 ];
-  };
-
-  i18n = {
-    defaultLocale = "en_US.UTF-8";
-    inputMethod.enable = true;
-    inputMethod.type = "ibus";
-  };
-
-  services = {
-    libinput.enable = true;
-    xserver = {
-      enable = true;
-      xkb.layout = "us";
-      xkb.variant = "altgr-intl";
-      xkb.options = "ctrl:nocaps";
-      wacom.enable = true;
-      videoDrivers = [ "modesetting" ];
-      modules = [ pkgs.xf86_input_wacom ];
-
-      displayManager.sx.enable = true;
-      windowManager.i3.enable = true;
-    };
-
-    udev.extraHwdb = ''
-      # not like this mattered at all
-      # we're not running udev from here
-    '';
-
-    udev.extraRules = ''
-      # ACTION=="add", SUBSYSTEM=="input", ...
-    '';
-  };
-
-  programs = {
-    light.enable = true;
-    wireshark = {
-      enable = true;
-      package = pkgs.wireshark-qt;
-    };
-    gnupg.agent = {
-      enable = true;
-    };
-  };
-
-  fonts.packages = with pkgs; [
-    font-awesome
-    noto-fonts
-    noto-fonts-cjk-sans
-    noto-fonts-emoji
-    noto-fonts-extra
-    dejavu_fonts
-    powerline-fonts
-    source-code-pro
-    cantarell-fonts
-  ];
-
-  users = {
-    mutableUsers = false;
-
-    users = {
-      user = {
-        isNormalUser = true;
-        group = "user";
-        extraGroups = [
-          "wheel"
-          "video"
-          "audio"
-          "dialout"
-          "users"
-          "kvm"
-          "wireshark"
-        ];
-        password = "unimportant";
-      };
-    };
-
-    groups = {
-      user = { };
-    };
-  };
-
-  security = {
-    pam.loginLimits = [
-      {
-        domain = "@audio";
-        item = "memlock";
-        type = "-";
-        value = "unlimited";
-      }
-      {
-        domain = "@audio";
-        item = "rtprio";
-        type = "-";
-        value = "99";
-      }
-      {
-        domain = "@audio";
-        item = "nofile";
-        type = "soft";
-        value = "99999";
-      }
-      {
-        domain = "@audio";
-        item = "nofile";
-        type = "hard";
-        value = "99999";
-      }
-    ];
-
-    sudo.extraRules = [
-      {
-        users = [ "user" ];
-        commands = [
-          {
-            command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
-            options = [ "NOPASSWD" ];
-          }
-        ];
-      }
-    ];
-  };
-
-  environment.systemPackages = with pkgs; [
-    a2jmidid
-    age
-    ardour
-    bemenu
-    blender
-    breeze-icons
-    breeze-qt5
-    bubblewrap
-    calf
-    claws-mail
-    darktable
-    duperemove
-    emacs
-    feh
-    file
-    firefox
-    fluidsynth
-    adwaita-icon-theme
-    gnuplot
-    graphviz
-    helm
-    i3status-rust
-    inkscape
-    jack2
-    jq
-    krita
-    ldns
-    libqalculate
-    libreoffice
-    man-pages
-    nix-diff
-    nix-index
-    nix-output-monitor
-    open-music-kontrollers.patchmatrix
-    pamixer
-    pavucontrol
-    pciutils
-    picom
-    pwgen
-    redshift
-    ripgrep
-    rlwrap
-    silver-searcher
-    soundfont-fluid
-    whois
-    wol
-    xclip
-    xdot
-    xdotool
-    xorg.xkbcomp
-    yt-dlp
-    zathura
-    borgbackup
-    linuxPackages.cpupower
-    mtr
-    kitty
-    xf86_input_wacom
-  ];
-
-  environment.pathsToLink = [ "/share/soundfonts" ];
-
-  systemd.user.services.run-python = {
-    after = [ "network-online.target" ];
-    script = ''
-      exec ${pkgs.python3}/bin/python
-    '';
-    serviceConfig = {
-      CapabilityBoundingSet = [ "" ];
-      KeyringMode = "private";
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      NoNewPrivileges = true;
-      PrivateDevices = true;
-      PrivateTmp = true;
-      PrivateUsers = true;
-      ProcSubset = "pid";
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = true;
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectKernelTunables = true;
-      ProtectProc = "invisible";
-      ProtectSystem = "strict";
-      RestrictAddressFamilies = "AF_INET AF_INET6";
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-        "@system-service"
-        "~ @resources @privileged"
-      ];
-      UMask = "077";
-    };
-  };
-
-  system.stateVersion = "23.11";
-}

contrib/OWNERS

@@ -0,0 +1 @@
+*

doc/OWNERS

@@ -0,0 +1 @@
+*

doc/manual/change-authors.yml

@@ -66,6 +66,9 @@ delan:
   forgejo: delan
   github: delan
 
+delroth:
+  github: delroth
+
 detroyejr:
   display_name: Jonathan De Troye
   github: detroyejr
@@ -74,6 +77,11 @@ edolstra:
   display_name: Eelco Dolstra
   github: edolstra
 
+emilazy:
+  display_name: Emily
+  forgejo: emilazy
+  github: emilazy
+
 ericson:
   display_name: John Ericson
   github: ericson2314
@@ -98,6 +106,9 @@ ian-h-chamberlain:
   forgejo: ian-h-chamberlain
   github: ian-h-chamberlain
 
+infinisil:
+  github: infinisil
+
 isabelroses:
   forgejo: isabelroses
   github: isabelroses
@@ -109,6 +120,11 @@ jade:
 just1602:
   forgejo: just1602
 
+kasimeka:
+  display_name: ورد
+  forgejo: janw4ld
+  github: kasimeka
+
 kfears:
   display_name: KFears
   forgejo: kfearsoff
@@ -147,14 +163,25 @@ ma27:
 matthewbauer:
   github: matthewbauer
 
+mic92:
+  github: mic92
+
 midnightveil:
   display_name: julia
   forgejo: midnightveil
   github: midnightveil
 
+nan-git:
+  display_name: NaN-git
+  github: NaN-git
+
 ncfavier:
   github: ncfavier
 
+not-my-profile:
+  display_name: Martin Fischer
+  github: not-my-profile
+
 p-e-meunier:
   display_name: Pierre-Etienne Meunier
   github: P-E-Meunier
@@ -226,6 +253,11 @@ vigress8:
   forgejo: vigress8
   github: vigress8
 
+vlinkz:
+  display_name: Victor Fuentes
+  forgejo: vlinkz
+  github: vlinkz
+
 winter:
   forgejo: winter
   github: winterqt
@@ -233,6 +265,9 @@ winter:
 xanderio:
   github: xanderio
 
+xokdvium:
+  github: xokdvium
+
 yorickvp:
   github: yorickvp
 

doc/manual/generate-manpage.nix

@@ -225,7 +225,7 @@ let
           showCategory = cat: ''
             ${optionalString (cat != "") "**${cat}:**"}
 
-            ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
+            ${listOptions (filterAttrs (n: v: v.category == cat && !v.hidden) allOptions)}
           '';
           listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
           showOption =

doc/manual/rl-next/arg-identifier.md

@@ -0,0 +1,16 @@
+---
+synopsis: First argument to `--arg`/`--argstr` must be a valid Nix identifier
+issues: [fj#496]
+category: "Breaking Changes"
+credits: [ma27]
+---
+
+The first argument to `--arg`/`--argstr` must be a valid Nix identifier, i.e.
+`nix-build --arg config.allowUnfree true` is now rejected.
+
+This is because that invocation is a false friend since it doesn't set
+`{ config = { allowUnfree = true; }; }`, but `{ "config.allowUnfree" = true; }`.
+
+The idea is to change the behavior to the latter in the long-term. For that,
+non-identifiers started giving a warning since 2.92 and are now rejected to give people
+who depend on that a chance to notice and potentially weigh in on the discussion.

doc/manual/rl-next/async-query.md

@@ -0,0 +1,20 @@
+---
+synopsis: "Improved susbtituter query speed"
+issues: []
+cls: []
+category: Improvements
+credits: [horrors]
+---
+
+The code used to query substituters for derivations has been rewritten slightly
+to take advantage of our asynchronous runtime. Such queries run for every build
+that could download from substituters and processes every derivation that isn't
+yet present on the local system. Previously Lix would use `http-connections` to
+limit query concurrency, even for modern caches that support HTTP/2 and have no
+limit on how many queries can be run concurrently on one single connection. Lix
+no longer does this, resulting in approximately 60% reduction in query time for
+medium-sized closures (e.g. NixOS system closures) during testing, although the
+exact number depends greatly on local network latency and generally improves as
+latency increases. Unlike previously setting `http-connections` to `1` or other
+low values no longer brings a massive penalty in query performance if the cache
+in use by the querying system supports HTTP/2 (as e.g. `cache.nixos.org` does).

doc/manual/rl-next/build-dir-mandatory.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`build-dir` no longer defaults to `temp-dir`"
+cls: [3453]
+category: "Fixes"
+credits: [horrors]
+---
+
+The directory in which temporary build directories are created no longer defaults
+to the value of the `temp-dir` setting to avoid builders making their directories
+world-accessible. This behavior has been used to escape the build sandbox and can
+cause build impurities even when not used maliciously. We now default to `builds`
+in `NIX_STATE_DIR` (which is `/nix/var/nix/builds` in the default configuration).

doc/manual/rl-next/certificate-authorities.md

@@ -0,0 +1,60 @@
+---
+synopsis: "Global certificate authorities are copied inside the builder's environment"
+issues: [gh#12698, fj#885]
+cls: [3765]
+category: Fixes
+credits: [raito, emilazy]
+---
+
+Previously, CA certificates were only installed at
+`/etc/ssl/certs/ca-certificates.crt` for sandboxed builds on Linux.
+
+This setup was insufficient in light of recent changes in `nixpkgs`, which now
+enforce HTTPS usage for `fetchurl`, even for fixed-output derivations, to
+mitigate confidentiality risks such as `netrc` or credentials leakage.
+`nixpkgs` still make use of a special package called `cacerts` which contains a
+copy of the CA certificates maintained by Nixpkgs and added as a reference for
+TLS-enabled fetchers.
+
+As a result, having a consistent and trusted certificate authority in all
+builder environments is becoming more essential.
+
+On `nix-darwin`, the `NIX_SSL_CERT_FILE` environment variable is always
+explicitly defined, but it is ignored by the sandbox setup.
+
+Simultaneously, Nix evaluates and propagates impure environment variables via
+`lib.proxyImpureEnvVars`, meaning that if `NIX_SSL_CERT_FILE` is set (which
+influences the default value for `ssl-cert-file`), it will be forwarded
+unchanged into the builder environment.
+
+However, on Linux, Nix also *copies* the CA file into the sandbox, creating a
+discrepancy between the value of `NIX_SSL_CERT_FILE` and the actual trusted
+certificate path used during the build.
+
+This divergence caused confusion and was partially addressed by attempts to
+whitelist the CA path in the Darwin sandbox (see cl/2906), but that approach
+involved a non-trivial path canonicalization step and is not as general as this one.
+
+To address this properly, we now emit a warning and override
+`NIX_SSL_CERT_FILE` inside the builder, explicitly pointing it to the CA file
+copied into the sandbox.
+
+This eliminates ambiguity between `NIX_SSL_CERT_FILE`
+and `ssl-cert-file`, ensuring consistent trust anchors across platforms.
+
+This warning might become a hard error as we figure out what to do regarding
+`lib.proxyImpureEnvVars` in nixpkgs.
+
+The behavior has been verified across sandboxed and unsandboxed builds on both
+Linux and Darwin.
+
+As a consequence of this change, approximately 500 KB of CA certificate data is
+now unconditionally copied into the build directory for fixed-output
+derivations.
+
+While this ensures consistent trust verification without having to restart the
+daemon after system upgrades, it may introduce a slight overhead in build
+performance. At present, no optimizations have been implemented to avoid this
+copy, but if this overhead proves noticeable in your workflows, please open an
+issue so we can evaluate and possibly implement different strategies to render
+trust anchors visible.

doc/manual/rl-next/cgroups-delegation.md

@@ -0,0 +1,69 @@
+---
+synopsis: New cgroup delegation model
+issues: [fj#537, fj#77]
+cls: [3230]
+category: "Breaking Changes"
+credits: [raito, horrors, lheckemann]
+---
+
+Builds using cgroups (i.e. `use-cgroups = true` and the experimental feature
+`cgroups`) now always delegate a cgroup tree to the sandbox.
+
+Compared to the original C++ Nix project, our delegation includes the
+`subtree_control` file as well, which means that the sandbox can disable
+certain controllers in its own cgroup tree.
+
+This is a breaking change because this requires the Nix daemon to run with an
+already delegated cgroup tree by the service manager.
+
+## How to setup the cgroup tree with systemd?
+
+systemd offers knobs to perform the required setup using:
+
+```
+[Service]
+Delegate=yes
+DelegateSubtree=supervisor
+```
+
+These directives are now included in our systemd packaging.
+
+## What about using Nix as root without connecting to the daemon?
+
+Builds run as `root` without connecting to the daemon relying on the cgroup
+feature are now broken, i.e.
+
+```console
+# nix-build --use-cgroups --sandbox ... # will not work
+```
+
+Consider doing instead:
+
+```console
+# systemd-run --same-dir --wait -p Delegate=yes -p DelegateSubgroup=supervisor nix-build --use-cgroups ...
+```
+
+If you need to disable cgroups temporarily, remember that you can do
+`NIX_CONF='include /etc/nix/nix.conf\nuse-cgroups = false' nix-build ...` or
+`nix-build --no-use-cgroups ...`.
+
+## What about other service managers than systemd?
+
+systemd has a [documentation](https://systemd.io/CGROUP_DELEGATION/) on how to
+handle cgroup delegation from service management perspective.
+
+If your service manager adheres to systemd semantics, e.g. writing an extended
+attribute `user.delegate=1` on the delegated cgroup tree directory and moving
+the `nix-daemon` process inside a cgroup tree to respect the inner process
+rule, then, the feature will work as well.
+
+## Why is the cgroup feature still experimental?
+
+While the cgroup feature unlocks many use cases, its behavior and integration (e.g. user experience), especially at scale on build farms or in multi-tenant environments, are not yet fully matured. There’s also potential for deeper systemd integration (e.g. using slices and scopes) that has not been fully explored.
+
+To avoid locking in an unstable interface, we’re keeping the experimental flag until we have validated the feature across a broader range of scenarios, including but not limited to:
+
+* Nix as root
+* Hydra-style build farms
+* Forgejo CI runners
+* Shared remote builders

doc/manual/rl-next/deprecated-derivation-types.md

@@ -0,0 +1,16 @@
+---
+synopsis: Deprecation of CA derivations, dynamic derivations, and impure derivations
+issues: [fj#815]
+cls: []
+significance: significant
+category: Miscellany
+credits: []
+---
+
+Content-addressed derivations are now deprecated and slated for removal in Lix 2.94.
+We're doing this because the CA derivation system has been a known cause of problems
+and inconsistencies, is unmaintained, habitually makes improving the store code very
+difficult (or blocks such improvements outright), and is beset by a number of design
+flaws that in our opinion cannot be fixed without a full reimplementation from zero.
+Dynamic derivations and impure derivations are built on the CA derivation framework,
+and owing to this they too are deprecated and slated for removal in another release.

doc/manual/rl-next/double-sigint.md

@@ -0,0 +1,25 @@
+---
+synopsis: "Hitting Control-C twice always terminates Lix"
+cls: [3574]
+issues: []
+category: "Improvements"
+credits: [horrors]
+---
+
+Hitting Control-C or sending `SIGINT` to Lix now prints an informational message
+if it is still running after on second, the second Control-C/`SIGINT` terminates
+Lix immediately without waiting for any shutdown code to finish running. Lix did
+not treat the second such event differently from first in the past; this made it
+impossible to easily terminate running Lix processes that got stuck in e.g. very
+expensive Nixlang code that never interacted with the store. We now terminate as
+soon as the user hits Control-C again without waiting any more, to much the same
+effect as putting Lix into the background and killing it immediately afterwards.
+
+This means you can now more conveniently break out of stuck Nixlang evaluations:
+```
+❯ nix-instantiate --eval --expr 'let f = n: if n == 0 then 0 else f (n - 1) + f (n - 1); in f 32'
+^CStill shutting down. Press ^C again to abort all operations immediately.
+^C
+
+❌130 ❯
+```

doc/manual/rl-next/exponential-backoff.md

@@ -0,0 +1,12 @@
+---
+synopsis: "libstore: exponential backoff for downloads"
+issues: [lix#932]
+cls: [3856]
+category: Fixes
+credits: [ma27]
+---
+
+The connection timeout when downloading from e.g. a binary cache is exponentially
+increased per failure. The option `connect-timeout` is now an alias to `max-connect-timeout`
+which is the maximum value for a timeout. The start value is controlled
+by `initial-connect-timeout` which is `5` by default.

doc/manual/rl-next/fix-develop-esc.md

@@ -0,0 +1,9 @@
+---
+synopsis: Fix develop shells for derivations with escape codes
+issues: [fj#991]
+cls: [4154, 4155]
+category: Fixes
+credits: [Qyriad]
+---
+
+ASCII control characters (including `\e`, used for ANSI escape codes) in derivation variables are now correctly escaped for `nix develop` and `nix print-dev-env`, instead of erroring.

doc/manual/rl-next/fix-gc-links.md

@@ -0,0 +1,13 @@
+---
+synopsis: "nix-store --delete: always remove obsolete hardlinks"
+issues: []
+cls: [3188]
+category: Fixes
+credits: [lheckemann]
+---
+
+Deleting specific paths using `nix-store --delete` or `nix store
+delete` previously did not delete hard links created by `nix-store
+--optimise` even if they became obsolete, unless _all_ of the given
+paths were deleted successfully. Now, hard links are always cleaned
+up, even if some of the given paths could not be deleted.

doc/manual/rl-next/fix-gc-stats.md

@@ -0,0 +1,24 @@
+---
+synopsis: "Report GC statistics correctly"
+issues: []
+cls: [3188]
+category: Fixes
+credits: [lheckemann]
+---
+
+Deleting specific paths using `nix-store --delete` or `nix store delete` previously did
+not report statistics correctly when some of the paths could not be deleted, even if
+others were deleted:
+
+```
+$ nix store delete /nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1 --delete-closure -v
+finding garbage collector roots...
+deleting '/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1'
+0 store paths deleted, 0.00 MiB freed
+error: Cannot delete some of the given paths because they are still alive. Paths not deleted:
+         k9bxzr1l92r5y6mihrkbpbr3fmc8qszx-libidn2-2.3.8
+         mbx9ii53lzjlrsnlrfmzpwm33ynljwdn-libunistring-1.3
+         rf8hcy6bldxdqc0g6q1dcka1vh47x69s-xgcc-14.2.1.20250322-libgcc
+         vbrdc5wgzn0w1zdp10xd2favkjn5fk7y-glibc-2.40-66
+       To find out why, use nix-store --query --roots and nix-store --query --referrers.
+```

doc/manual/rl-next/flake-self-attrs.md

@@ -0,0 +1,16 @@
+---
+synopsis: Add `inputs.self.submodules` flake attribute
+issues: [fj#942]
+cls: [3839]
+category: Features
+credits: [edolstra, kasimeka]
+---
+
+A port of <https://github.com/NixOS/nix/pull/12421> to Lix, which:
+
+- adds a general `inputs.self` flake attribute that retroactively applies
+  configurations to a flake after it's been fetched, then triggers a refetch of
+  the flake with the new config.
+- implements `inputs.self.submodules` that allows a flake to declare its need
+  for submodules, which are then fetched automatically with no need to pass
+  `?submodules=1` anywhere.

doc/manual/rl-next/hyperlinks-in-attrsets.md

@@ -0,0 +1,16 @@
+---
+synopsis: Add hyperlinks in attr set printing
+issues: []
+cls: [3790]
+category: Features
+credits: [jade]
+---
+
+The attribute set printer, such as is seen in `nix repl` or in type errors, now prints hyperlinks on each attribute name to its definition site if it is known.
+
+Example: all of the attributes shown here are hyperlinks to the exact definition site of the attribute in question:
+
+```
+$ nix eval -f '<nixpkgs>' lib.licenses.mit
+{ deprecated = false; free = true; fullName = "MIT License"; redistributable = true; shortName = "mit"; spdxId = "MIT"; url = "https://spdx.org/licenses/MIT.html"; }
+```

doc/manual/rl-next/ignore-try-default.md

@@ -0,0 +1,24 @@
+---
+synopsis: Repl debugger uses `--ignore-try` by default
+issues: [lix#666]
+cls: [3488]
+category: Breaking Changes
+credits: [jade]
+---
+Previously, using the debugger meant that exceptions thrown in `builtins.tryEval` would trigger the debugger.
+
+However, this caught nixpkgs initialization code, which is unhelpful in the majority of cases, so we changed the default.
+
+To get the old behaviour, use `--no-ignore-try`.
+
+```
+$ nix repl --debugger --expr 'with import <nixpkgs> {}; pkgs.hello'
+Lix 2.94.0-dev-pre20250625-9a59106
+Type :? for help.
+error: file 'nixpkgs-overlays' was not found in the Nix search path (add it using $NIX_PATH or -I)
+
+This exception occurred in a 'tryEval' call. Use --ignore-try to skip these.
+
+Added 13 variables.
+nix-repl>
+```

doc/manual/rl-next/infallible-build-dirs.md

@@ -0,0 +1,25 @@
+---
+synopsis: "Fallback to safe temp dir when build-dir is unwritable"
+issues: [fj#876]
+cls: [3501]
+category: "Fixes"
+credits: ["raito", "horrors"]
+---
+
+Non-daemon builds started failing with a permission error after introducing the `build-dir` option:
+
+```
+$ nix build --store ~/scratch nixpkgs#hello --rebuild
+error: creating directory '/nix/var/nix/builds/nix-build-hello-2.12.2.drv-0': Permission denied
+```
+
+This happens because:
+
+1. These builds are not run via the daemon, which owns `/nix/var/nix/builds`.
+2. The user lacks permissions for that path.
+
+We considered making `build-dir` a store-level option and defaulting it to `<chroot-root>/nix/var/nix/builds` for chroot stores, but opted instead for a fallback: if the default fails, Nix now creates a safe build directory under `/tmp`.
+
+To avoid CVE-2025-52991, the fallback uses an extra path component between `/tmp` and the build dir.
+
+**Note**: this fallback clutters `/tmp` with build directories that are not cleaned up. To prevent this, explicitly set `build-dir` to a path managed by Lix, even for local workloads.

doc/manual/rl-next/integer-coercion.md

@@ -0,0 +1,47 @@
+---
+synopsis: Experimental integer coercion in interpolated strings
+issues: []
+cls: [3198]
+category: "Features"
+credits: [raito, delroth, horrors, winter]
+---
+
+Ever tried interpolating a port number in Lix and ended up with something like this?
+
+```nix
+"http://${config.network.host}:${builtins.toString config.network.port}/"
+```
+
+You're not alone. Thousands of Lix users suffer every day from excessive `builtins.toString` syndrome. It’s 2025, and we still have to cast integers to use them in strings.
+
+To address this, Lix introduces the **`coerce-integers`** experimental feature. When enabled, interpolated integers within `"${...}"` are automatically coerced to strings. This allows writing:
+
+```nix
+"http://${config.network.host}:${config.network.port}/"
+```
+
+without additional conversion.
+
+To enable the feature, you need to add `coerce-integers` to your set of experimental features.
+
+### Stabilization criteria
+
+The `coerce-integers` feature is experimental and limited strictly to string interpolation (`"${...}"`). Before stabilization, the following must hold:
+
+1. **Interpolation-only**
+   Coercion must not occur outside interpolation. Expressions like `"" + 42` must continue to fail.
+
+2. **Expectation that no explicit cast are being observed**
+   Cases observing explicit coercion (e.g., via `tryEval` gadget or similar) are expected not to be load-bearing in actual production code.
+
+### Timeline for stabilization
+
+If the feature proves safe and is widely adopted across typical usage (e.g., actual configurations in the wild turning on the flag, non-trivial out-of-tree projects using it), the experimental flag will be removed **after six months of active use or two Lix releases**, whichever is longer.
+
+This avoids locking the feature in experimental status indefinitely, as happened with Flakes, while allowing time for validation and ecosystem integration.
+
+### What about coercing floats or more?
+
+Coercion beyond integers -- such as for floats or other types -- is **not planned**, even under an experimental flag. Questions like "what is the canonical string representation of a float?" involve subtle and context-dependent trade-offs. Without a robust and principled mechanism to define and audit such behavior, introducing broader coercion risks setting unintended and hard-to-reverse precedents. The scope of `coerce-integers` is intentionally narrow and will remain so.
+
+In terms of outlook, a proposal like https://git.lix.systems/lix-project/lix/issues/835 could pave the way for a better solution.

doc/manual/rl-next/json-number-overflow.md

@@ -0,0 +1,16 @@
+---
+synopsis: Parse overflowing JSON number literals as floating‐point
+issues: []
+cls: [3919]
+category: "Fixes"
+credits: [emilazy]
+---
+
+Previously, `builtins.fromJSON "-9223372036854775809"` would
+return a floating‐point number, while `builtins.fromJSON
+"9223372036854775808"` would cause an evaluation error. This was
+introduced with the banning of integer overflow in Lix 2.91; previously
+the latter would result in C++ undefined behaviour. These cases are
+now treated consistently with JSON’s model of a single numeric type,
+and JSON number literals that do not fit in a Nix‐language integer
+will be parsed as floating‐point numbers.

doc/manual/rl-next/keep-failed-chown.md

@@ -0,0 +1,13 @@
+---
+synopsis: "`--keep-failed` chowns the build directory to the user that request the build"
+issues: []
+cls: []
+category: Improvements
+credits: [horrors]
+---
+
+Running a build with `--keep-failed` now chowns the temporary directory from the
+builder user and group to the user that request the build if the build came from
+a local user connected to the daemon. This makes inspecting failed derivations a
+lot easier. On Linux the build directory made visible to the user will not be in
+the same path as it was in the sandbox and continuing builds will usually break.

doc/manual/rl-next/log-output-terminal-codes.md

@@ -0,0 +1,11 @@
+---
+synopsis: Fix handling of OSC codes in terminal output
+issues: [fj#160]
+cls: [3143]
+category: Fixes
+credits: [lilyball]
+---
+
+OSC codes in terminal output are now handled correctly, where OSC 8 (hyperlink) is preserved any
+time color codes are allowed and all other OSC codes are stripped out. This applies not only to
+output from build commands but also to rendered documentation in the REPL.

doc/manual/rl-next/mismatch-check-path.md

@@ -0,0 +1,14 @@
+---
+synopsis: Better debuggability on fixed-output hash mismatches
+issues: []
+cls: []
+category: Improvements
+credits: [lheckemann]
+---
+
+Fixed-output derivation hash mismatch error messages will now include the path that was
+produced unexpectedly, and this path will be registered as valid even if `--check`
+(`nix-store`, `nix-build`) or `--rebuild` (`nix build`) was passed. This makes comparing
+the expected path with the obtained path easier, and is useful for debugging when
+upstreams modify previously-published releases or when changes in fixed-output
+derivations' dependencies affect their output unexpectedly.

doc/manual/rl-next/nej-no-instantiate.md

@@ -0,0 +1,10 @@
+---
+synopsis: "nix-eval-jobs: support `--no-instantiate` flag"
+issues: [fj#987]
+category: Features
+credits: [mic92,ma27]
+---
+
+`nix-eval-jobs` now supports a flag called `--no-instantiate`. With this enabled,
+no write operations on the eval store are performed. That means, only evaluation is
+performed, but derivations (and their gcroots) aren't created.

doc/manual/rl-next/nix-develop-output-checks.md

@@ -0,0 +1,29 @@
+---
+synopsis: "Fix nix develop for derivations that rejects dependencies with structured attrs"
+issues: [fj#997]
+cls: [4182]
+category: Fixes
+credits: [raito]
+---
+
+For the sake of concision, we refer to `disallowedReferences` in what follows,
+but all output checks were equally fixed:
+`{dis,}allowed{References,Requisites}`.
+
+Derivations can define *output checks* to reject unwanted dependencies, such as
+interpreters like `bash` or compilers like `gcc`. This can be done in two ways:
+
+* **Legacy style**: `disallowedReferences = [ ... ]` in the environment.
+* **Structured attrs**: `outputChecks.<output>.disallowedReferences = [ ... ]`,
+  typically used in `__json`.
+
+Only the structured form supports derivations with multiple outputs.
+
+`nix develop` internally rewrites derivations to create development shells. It
+relied on the legacy `disallowedReferences`, and failed to honor the structured
+variant. This led to broken shells in cases where `bashInteractive` was
+explicitly disallowed using structured output checks, e.g. `nix develop
+nixpkgs#systemd` after the "bash-less NixOS" changes.
+
+This fix teaches `nix develop` to respect structured output checks, restoring
+support for such derivations.

doc/manual/rl-next/nix-instantiate-raw.md

@@ -0,0 +1,12 @@
+---
+synopsis: "Add --raw flag to `nix-instantiate --eval` for unescaped output"
+issues: []
+prs: [gh#12119]
+cls: [2886]
+category: Improvements
+credits: [not-my-profile, infinisil, raito]
+---
+
+The `nix-instantiate --eval` command now supports a `--raw` flag. When used,
+the result must be coercible to a string (as with `${...}`) and is printed
+verbatim, without quotes or escaping.

doc/manual/rl-next/nix-store-ls-read-nar-listings.md

@@ -0,0 +1,12 @@
+---
+synopsis: Allow `nix store ls` to read nar listings from binary cache stores.
+issues: []
+cls: [3225]
+category: Improvements
+credits: [vlinkz]
+---
+
+The `nix store ls` command now supports reading `.ls` nar listings from binary cache stores.
+If a listing is detected for the store path being queried, the nar is no longer downloaded.
+These nar listings are available in binary cache stores where the `write-nar-listing` option is
+enabled, such as cache.nixos.org.

doc/manual/rl-next/nix_path-nix-eval-jobs.md

@@ -0,0 +1,10 @@
+---
+synopsis: "nix-eval-jobs: retain NIX_PATH"
+issues: []
+cls: [3859]
+category: Fixes
+credits: [ma27,mic92]
+---
+
+`nix-eval-jobs` doesn't clear the `NIX_PATH` from the environment anymore. This matches the behavior
+of [upstream version `2.30`](https://github.com/nix-community/nix-eval-jobs/releases/tag/v2.30.0).

doc/manual/rl-next/output-cycle-refs.md

@@ -0,0 +1,25 @@
+---
+synopsis: "show tree with references that lead to an output cycle"
+issues: [fj#551]
+category: Improvements
+credits: [ma27]
+---
+
+When Lix determines a cyclic dependency between several outputs of a derivation,
+it now displays which files in which outputs lead to an output cycle:
+
+```
+error: cycle detected in build of '/nix/store/gc5h2whz3rylpf34n99nswvqgkjkigmy-demo.drv' in the references of output 'bar' from output 'foo'.
+
+       Shown below are the files inside the outputs leading to the cycle:
+       /nix/store/3lrgm74j85nzpnkz127rkwbx3fz5320q-demo-bar
+       └───lib/libfoo: …stuffbefore /nix/store/h680k7k53rjl9p15g6h7kpym33250w0y-demo-baz andafter.…
+           → /nix/store/h680k7k53rjl9p15g6h7kpym33250w0y-demo-baz
+           └───share/snenskek: …???? /nix/store/dm24c76p9y2mrvmwgpmi64rryw6x5qmm-demo-foo ....…
+               → /nix/store/dm24c76p9y2mrvmwgpmi64rryw6x5qmm-demo-foo
+               └───bin/alarm: …textexttext/nix/store/3lrgm74j85nzpnkz127rkwbx3fz5320q-demo-bar abcabcabc.…
+                   → /nix/store/3lrgm74j85nzpnkz127rkwbx3fz5320q-demo-bar
+```
+
+Please note that showing the files and its contents while displaying the cycles only works
+on Linux.

doc/manual/rl-next/pasta.md

@@ -0,0 +1,20 @@
+---
+synopsis: "Fixed output derivations can be run using `pasta` network isolation"
+cls: [3452]
+issues: [fj#285]
+category: "Breaking Changes"
+credits: [horrors, puck]
+---
+
+Fixed output derivations traditionally run in the host network namespace.
+On Linux this allows such derivations to communicate with other sandboxes
+or the host using the abstract Unix domains socket namespace; this hasn't
+been unproblematic in the past and has been used in two distinct exploits
+to break out of the sandbox. For this reason fixed output derivations can
+now run in a network namespace (provided by [`pasta`]), restricted to TCP
+and UDP communication with the rest of the world. When enabled this could
+be a breaking change and we classify it as such, even though we don't yet
+enable or require such isolation by default. We may enforce this in later
+releases of Lix once we have sufficient confidence that breakage is rare.
+
+[`pasta`]: https://passt.top/

doc/manual/rl-next/reliance-on-bash-for-ssh.md

@@ -0,0 +1,21 @@
+---
+synopsis: Remove reliance on Bash for remote stores via SSH
+issues: [fj#830, fj#805, fj#304]
+cls: [3159]
+category: "Fixes"
+credits: [raito]
+---
+
+The pre-flight `echo started` handshake -- added years ago to catch race conditions -- has been removed.
+
+After removal of connection sharing in Lix 2.93, it required a Bash-compatible shell and a standard `echo`, so it failed on:
+
+  * builders protected by `ForceCommand` wrappers (e.g. `nix-remote-build`),
+  * BusyBox / initrd images with no Bash,
+  * hosts using non-POSIX shells such as Nushell.
+
+The race the probe once addressed was tied to SSH connection-sharing -- since connection-sharing code has already been removed, the probe is now pointless.
+
+Real connection or protocol errors are now left to SSH/Nix to report directly.
+
+This is technically a breaking change if you had scripts that relied on the literal "started" which needs to be updated to rely on other signals, e.g., exit codes.

doc/manual/rl-next/remove-old-wires.md

@@ -0,0 +1,20 @@
+---
+synopsis: Remove support for daemon protocols before 2.18
+issues: [fj#510]
+cls: [3249]
+significance: significant
+category: "Breaking Changes"
+credits: [horrors]
+---
+
+Support for daemon wire protocols belonging to Nix 2.17 or older have been
+removed. This impacts clients connecting to the local daemon socket or any
+remote builder configured using the `ssh-ng` protocol. Builders configured
+with the `ssh` protocol are still accessible from clients such as Nix 2.3.
+Additionally Lix will not be able to connect to an old daemon locally, and
+remote build connections to old daemons is likewise limited to `ssh` urls.
+
+We have decided to take this step because the old protocols are very badly
+tested (if at all), maintenance overhead is high, and a number of problems
+with their design makes it infeasible to remain backwards compatible while
+we move Lix to a more modern RPC mechanism with better versioning support.

doc/manual/rl-next/remove-write-to.md

@@ -0,0 +1,14 @@
+---
+synopsis: "`nix eval --write-to` has been removed"
+cls: [4045]
+issues: [fj#974, fj#227]
+category: "Breaking Changes"
+credits: [horrors]
+---
+
+`nix eval --write-to` has been removed since it was underspecified, not widely
+useful, and prone to security-sensitive misbehaviors. The feature was added in
+Nix 2.4 purely for internal use in the build system. According to our research
+it hasn't found any use outside of some distribution packaging scripts. Please
+use structured outputs formats (such as JSON) instead as they have better type
+fidelity, don't conflate attributes with paths, and are useful to other tools.

doc/manual/rl-next/remove-xp-derivations.md

@@ -0,0 +1,17 @@
+---
+synopsis: Remove impure derivations and dynamic derivations
+issues: [fj#815]
+cls: [3210]
+significance: significant
+category: "Breaking Changes"
+credits: [horrors]
+---
+
+The `impure-derivations` and `dynamic-derivations` experimental feature have
+been removed.
+
+New impure or dynamic derivations cannot be created from this point forward, and
+any such pre-existing store derivations canot be read or built any more.
+Derivation outputs created by building such a derivation are still valid
+until garbage collected; existing store derivations can only be garbage
+collected.

doc/manual/rl-next/remove-xp-parse-toml-timestamps.md

@@ -0,0 +1,17 @@
+---
+synopsis: Remove the `parse-toml-timestamps` experimental feature
+category: "Breaking Changes"
+credits: [emilazy]
+---
+
+The `parse-toml-timestamps` experimental feature has been removed.
+
+This feature used in‐band signalling to mark timestamps, making it
+impossible to unambiguously parse TOML documents. It also exposed
+implementation‐defined behaviour in the TOML specification that
+changed in the toml11 parser library.
+
+Any interface for parsing TOML timestamps suitable for future
+stabilization would necessarily involve breaking changes, and there
+is no evidence this experimental feature is being relied upon in the
+wild, so it has been removed.

doc/manual/rl-next/repl-overlays-pure.md

@@ -0,0 +1,20 @@
+---
+synopsis: "repl-overlays now work in the debugger for flakes"
+issues: [fj#777]
+cls: [3398]
+category: Fixes
+credits: [jade]
+---
+Due to a bug, it was previously not possible to use the debugger on flakes with repl-overlays, or with pure evaluation in general:
+
+```
+$ nix repl --pure-eval
+Lix 2.94.0-dev-pre20250617-87d99da
+Type :? for help.
+Loading 'repl-overlays'...
+error: access to absolute path '/Users/jade/.config/nix/repl.nix' is forbidden in pure eval mode (use '--impure' to override)
+```
+
+This is now fixed.
+The contents of the repl-overlays file itself (i.e. most typically the top level lambda in it) will be evaluated in impure mode.
+It may be necessary to use `builtins.seq` to force the impure operations to happen first if one wants to do impure operations inside a repl-overlays file in pure evaluation mode.

doc/manual/rl-next/report-forbidden-path-as-chain.md

@@ -0,0 +1,19 @@
+---
+synopsis: "`disallowedRequisites` now reports chains of disallowed requisites"
+issues: [fj#334,fj#626,gh#10877]
+category: Improvements
+credits: [ma27,roberth]
+---
+
+When a build fails because of [`disallowedRequisites`](@docroot@/language/advanced-attributes.md#adv-attr-disallowedRequisites), the error message now includes the chain of references that led to the failure. This makes it easier to see in which derivations the chain can be broken, to resolve the problem.
+
+Example:
+
+```
+$ nix-build -A hello
+error: output '/nix/store/0b7k85gg5r28gb54px9nq7iv5986mns9-hello-2.12.2' is not allowed to refer to the following paths:
+       /nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-glibc-2.40-66
+       Shown below are chains that lead to the forbidden path(s).
+       /nix/store/0b7k85gg5r28gb54px9nq7iv5986mns9-hello-2.12.2
+       └───/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-glibc-2.40-66
+```

doc/manual/rl-next/snix-http-cache-integration.md

@@ -0,0 +1,17 @@
+---
+synopsis: "libstore/binary-cache-store: don't cache narinfo on nix copy, remove negative entry"
+issues: []
+cls: [3789]
+category: Fixes
+credits: [ma27]
+---
+
+When using e.g. [Snix's nar-bridge](https://snix.dev/docs/components/overview/#nar-bridge) via
+an `http`-store, Lix would create cache entries with a wrong URL to the NAR when uploading
+a store-path.
+
+This caused hard build failures for Hydra.
+
+Lix doesn't create these entries on upload anymore. Instead, it only removes negative cache entries.
+The cache entry for a narinfo is now created the first time, Lix queries the cache
+for the previously uploaded store-path again.

doc/manual/rl-next/static-linking.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Lix libraries can now be linked statically"
+issues: [fj#789]
+cls: [3775, 3778]
+category: Fixes
+credits: [alois31]
+---
+Previously the pkg-config files distributed with Lix were only suitable for dynamic linkage, causing "undefined reference to…" linker errors when trying to link statically.
+Private dependency information has now been added to make static linkage work as expected without user intervention.
+In addition, relevant static libraries are now prelinked to avoid strange failures due to missing static initializers.

doc/manual/rl-next/symbol-value-reuse.md

@@ -0,0 +1,34 @@
+---
+synopsis: Symbols reuses once-allocated Value to reduce garbage collected allocations
+issues: []
+cls: [3308, 3300, 3314, 3310, 3312, 3313]
+category: Improvements
+credits: [raito, horrors, thubrecht, xokdvium, nan-git]
+---
+
+In the Lix evaluator, **symbols** represent immutable strings, like those used
+for attribute names.
+
+In evaluator design, such strings are typically [**interned**](https://en.wikipedia.org/wiki/String_interning), stored uniquely
+to save memory, and Lix inherits this approach from the original C++ codebase.
+
+However, some builtins, like `builtins.attrNames`, must return a `Value` type
+that can represent any Nix value (strings, integers, lists, etc.).
+
+Before this change, these builtins would create lists of `Value` objects by
+allocating them through the garbage collector, copying the symbol’s string
+content each time.
+
+This allocation is unnecessary if the interned symbols themselves also hold a
+`Value` representation allocated outside the garbage collector, since these
+live for the full duration of evaluation.
+
+As a result, this reduces the number of allocations, leading to:
+
+* A significant drop in maximum [resident set memory](https://en.wikipedia.org/wiki/Resident_set_size) (RSS), with some large-scale
+  tests showing up to 11% (about 500 MiB) savings in large colmena deployments.
+* A slight decrease in CPU usage during Nix evaluations.
+
+This change is inspired by https://github.com/NixOS/nix/pull/13258 but the approach is different.
+
+**Note** : [`xokdvium`](https://github.com/xokdvium) is the rightful author of https://gerrit.lix.systems/c/lix/+/3300 and the credit was missed on our end during the development process. We are deeply sorry for this mistake.

doc/manual/rl-next/toml-number-overflow.md

@@ -0,0 +1,14 @@
+---
+synopsis: Reject overflowing TOML integer literals
+issues: []
+cls: [3916]
+category: "Breaking Changes"
+credits: [emilazy]
+---
+
+The toml11 library used by Lix was updated. The new
+version aligns with the [TOML v1.0.0 specification’s
+requirement](https://toml.io/en/v1.0.0#integer) to reject integer
+literals that cannot be losslessly parsed. This means that code like
+`builtins.fromTOML "v=0x8000000000000000"` will now produce an error
+rather than silently saturating the integer result.

doc/manual/rl-next/uid-range-requires-cgroups.md

@@ -0,0 +1,15 @@
+---
+synopsis: uid-range depends on cgroups
+issues: []
+cls: [3230]
+category: "Breaking Changes"
+credits: [raito, horrors]
+---
+
+`uid-range` builds now depends on `cgroups`, an experimental feature.
+
+`uid-range` builds already depended upon `auto-allocate-uids`, another experimental feature.
+
+The rationale for doing so is that `uid-range` provides a sandbox with many
+UIDs, this is useful for re-mapping them into a nested namespace, e.g. a
+container.

doc/manual/rl-next/zsh-completions-desc.md

@@ -0,0 +1,10 @@
+---
+synopsis: add description to zsh completions
+issues: [fj#910]
+cls: [3632]
+category: "Fixes"
+credits: [matthewbauer]
+---
+
+Emit descriptions when completing args in zsh completions. This uses the descriptions we already
+provided in NIX\_GET\_COMPLETIONS.

doc/manual/src/SUMMARY.md

@@ -20,7 +20,6 @@
   - [Basic Package Management](package-management/basic-package-mgmt.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
-    - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)

doc/manual/src/command-ref/nix-env/install.md

@@ -148,8 +148,8 @@ To copy the store path with symbolic name `gcc` from another profile:
 $ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc
 ```
 
-To install a specific [store derivation] (typically created by
-`nix-instantiate`):
+To install a specific [store derivation](@docroot@/glossary.md#gloss-store-derivation)
+(typically created by `nix-instantiate`):
 
 ```console
 $ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv

doc/manual/src/command-ref/nix-instantiate.md

@@ -5,7 +5,7 @@
 # Synopsis
 
 `nix-instantiate`
-  [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ]
+  [`--parse` | `--eval` [`--strict`] [`--raw`] [`--json`] [`--xml`] ]
   [`--read-write-mode`]
   [`--arg` *name* *value*]
   [{`--attr`| `-A`} *attrPath*]
@@ -107,15 +107,27 @@ See that section for complete details (`nix-build --help`), but in summary, a pa
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
+  - `--raw`\
+    When used with `--eval`, the result must be coercible to a string, i.e.,
+    something that can be converted using `${...}`.
+
+    Integers will always generate an error when output via `--raw`, regardless of
+    [`coerce-integers`](../contributing/experimental-features.md#xp-feature-coerce-integers) being enabled, to avoid ambiguity.
+
+    The output is printed exactly as-is, with no quotes, escaping, or trailing
+    newline.
+
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
-    representation of the abstract syntax tree rather than as a Nix expression.
+    representation of the resulting value rather than as a Nix expression.
+
+    The conversion behaviour, if `--strict` is passed, is the same as
+    [`builtins.toJSON`](../language/builtins.md#builtins-toJSON).
 
   - `--xml`\
     When used with `--eval`, print the resulting value as an XML
-    representation of the abstract syntax tree rather than as a Nix expression.
-    The schema is the same as that used by the [`toXML`
-    built-in](../language/builtins.md).
+    representation of the resulting value rather than as a Nix expression.
+    The schema is the same as that used by [`builtins.toXML`](../language/builtins.md#builtins-toXML).
 
   - `--read-write-mode`\
     When used with `--eval`, perform evaluation in read/write mode so

doc/manual/src/command-ref/nix-store/realise.md

@@ -15,7 +15,6 @@ Each of *paths* is processed as follows:
   1. If it is not [valid], substitute the store derivation file itself.
   2. Realise its [output paths]:
     - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure].
-      - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database].
     - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. <!-- TODO: Link to build process page #8888 -->
 - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters].
 
@@ -28,7 +27,6 @@ If no substitutes are available and no store derivation is given, realisation fa
 [store objects]: @docroot@/glossary.md#gloss-store-object
 [closure]: @docroot@/glossary.md#gloss-closure
 [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters
-[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations
 [Nix database]: @docroot@/glossary.md#gloss-nix-database
 
 The resulting paths are printed on standard output.

doc/manual/src/contributing/cli-guideline.md

@@ -661,8 +661,8 @@ Verbosity levels are:
 
 The default level that the command starts is `ERROR`. The simplest way to
 increase the verbosity by stacking `-v` option (eg: `-vvv == level 3 == INFO`).
-There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and
-`--quiet` to run in `ERROR` verbosity level.
+Use `--quiet` to decrease verbosity by one level.
+There is one shortcut, `--debug` to run in `DEBUG` verbosity level.
 
 ----------
 

doc/manual/src/contributing/hacking.md

@@ -11,7 +11,19 @@ The following instructions assume you already have some version of Nix or Lix in
 
 [installation instructions]: ../installation/installation.md
 
-## Building Lix in a development shell
+A typical development flow for simple changes in Lix looks like:
+- [Set up and build Lix](#building)
+- For large changes, check in regarding design and possibly create an RFD issue on Forgejo
+- Make the changes in your editor
+- [Send the changes to Gerrit](#sending-to-gerrit)
+- Once you have the number for the CL from Gerrit to put in the changelog, [write a changelog entry](#release-notes) and amend it into the commit
+- Update the Gerrit change by submitting it with the same command as the first time
+- Request and receive a code review
+- Address feedback from the review
+- Amend commits, send to Gerrit again
+- Submit the approved change
+
+## Building Lix in a development shell {#building}
 
 ### Setting up the development shell
 
@@ -39,7 +51,7 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-Run a clean build and test with `just clean build install test`.
+Run a clean build and test with `just clean setup build install test`.
 
 You can also run the unit tests and integration tests separately:
 
@@ -48,7 +60,7 @@ $ just setup build test-unit
 $ just install test-integration
 ```
 
-Many targets have a `-custom` variant which pass extra arguments to `meson`.
+Many justfile aliases have a `-custom` variant which pass extra arguments to `meson`.
 For example, to work on both Lix and nix-eval-jobs you can run:
 
 ```
@@ -129,7 +141,36 @@ To inspect the canonical source of truth on what the state of the buildsystem co
 $ meson introspect
 ```
 
-## Building Lix outside of development shells
+## Sending changes to Gerrit for review {#sending-to-gerrit}
+
+We use Gerrit for all our code review in Lix.
+Our instance is at <https://gerrit.lix.systems>.
+
+There's much more information about how to use Gerrit in the [wiki section on Gerrit][wiki-gerrit] including how to use Jujutsu, how to use the UI and more.
+The Snix project also has some Gerrit information [in their contributing docs][snix-gerrit].
+
+[wiki-gerrit]: https://wiki.lix.systems/books/lix-contributors/chapter/gerrit
+[snix-gerrit]: https://snix.dev/docs/guides/contributing/
+
+The gist is that once you have your SSH key and git remote set up, you can send commits for review with:
+
+```
+$ git remote set-url origin ssh://YOURUSERNAME@gerrit.lix.systems:2022/lix
+$ git push origin HEAD:refs/for/main
+```
+
+Then, you can request a review via the "Reply" button on the web UI.
+If you click "Suggest Owners", it will try to suggest the maintainers of the area of the code change to send review requests to.
+Requesting reviews from multiple people is normal.
+
+We do our best to respond to directly sent reviews in a few days, so feel free to request another reviewer or ask on Matrix if you've not got a response for a while.
+Keep in mind that Lix is a volunteer project and we have limited bandwidth, so some changes aren't feasible to shepherd through; please check in on Matrix at design time when doing large changes.
+
+Once you get a `Code-Review+2` vote on your change, it's rebased on `main` and CI marks it `Verified+1`, you're able (and usually expected, so you can have a second chance to check it over) to hit the Submit button to merge it.
+If the change appears as "Rebase Required", you need to rebase it on `main` locally or via the Gerrit UI and wait for `Verified+1` before the Submit button is made active
+The `Code-Review+2` from before will stick around through trivial rebases so no need to re-request review for a mere rebase.
+
+## Building Lix with `nix`
 
 To build a release version of Lix for the current operating system and CPU architecture:
 
@@ -286,10 +327,10 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
 
-### Checking links in the manual
+# Manual and documentation
+
+## Building the manual
 
-The build checks for broken internal links.
-This happens late in the process, so `nix build` is not suitable for iterating.
 To build the manual incrementally, run:
 
 ```console
@@ -301,15 +342,20 @@ meson compile -C build manual
 [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck
 [URI fragments]: https://en.wikipedia.org/wiki/URI_fragment
 
-#### `@docroot@` variable
+The built manual is in `build/doc/manual/manual/index.html`.
 
-`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.
+The build checks for broken internal links.
+This happens late in the process, so `nix build` is not suitable for iterating and it's recommended to use the `meson` command above instead.
 
-If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory.
+### `@\docroot\@` variable
 
-If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it.
-See existing `@docroot@` logic in the [Makefile].
-Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`.
+`@\docroot\@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.
+
+If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@\docroot\@` to reference the `doc/manual/src` directory.
+
+If the `@\docroot\@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@\docroot\@` replacement needs to be applied to the generated source file that mentions it.
+See existing `@\docroot\@` logic in `doc/manual/substitute.py`.
+Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@\docroot\@`.
 
 ## API documentation
 
@@ -341,7 +387,7 @@ You can build it yourself:
 
 Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)).
 
-## Add a release note
+## Add a release note {#release-notes}
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.
 
@@ -410,15 +456,15 @@ The following properties are supported:
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
-Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
+Development releases have a generated `rl-next.md`.
 
-## Adding experimental or deprecated features, global settings, or builtins
+# Adding experimental or deprecated features, global settings, or builtins
 
 Experimental and deprecated features, global settings, and builtins are generally referenced both in the code and in the documentation.
 To prevent duplication or divergence, they are defined in data files, and a script generates the necessary glue.
 The data file format is similar to the release notes: it consists of a YAML metadata header, followed by the documentation in Markdown format.
 
-### Experimental or deprecated features
+## Experimental or deprecated features
 
 Experimental and deprecated features support the following metadata properties:
 * `name` (required): user-facing name of the feature, to be used in `nix.conf` options and on the command line.
@@ -428,7 +474,7 @@ Experimental and deprecated features support the following metadata properties:
 Experimental feature data files should live in `lix/libutil/experimental-features`, and deprecated features in `lix/libutil/deprecated-features`.
 They must be listed in the `experimental_feature_definitions` or `deprecated_feature_definitions` lists in `lix/libutil/meson.build` respectively to be considered by the build system.
 
-### Global settings
+## Global settings
 
 Global settings support the following metadata properties:
 * `name` (required): user-facing name of the setting, to be used as key in `nix.conf` and in the `--option` command line argument.
@@ -456,7 +502,7 @@ Settings are not collected in a single place in the source tree, so an appropria
 Look for related setting definition files under second-level subdirectories of `lix` whose name includes `settings`.
 Then add the new file there, and don't forget to register it in the appropriate `meson.build` file.
 
-### Builtin functions
+## Builtin functions
 
 The following metadata properties are supported for builtin functions:
 * `name` (required): the language-facing name (as a member of the `builtins` attribute set) of the function.
@@ -472,7 +518,7 @@ The following metadata properties are supported for builtin functions:
 
 New builtin function definition files must be added to `lix/libexpr/builtins` and registered in the `builtin_definitions` list in `lix/libexpr/meson.build`.
 
-### Builtin constants
+## Builtin constants
 The following metadata properties are supported for builtin constants:
 * `name` (required): the language-facing name (as a member of the `builtins` attribute set) of the constant.
 * `type` (required): the Nix language type of the constant; the C++ type is automatically derived.

doc/manual/src/contributing/testing.md

@@ -449,9 +449,6 @@ I grepped `lix/` for `get[eE]nv\("` to find the mentions in Lix code.
 - `NIX_CLIENT_PACKAGE` - Runs the test suite against an alternate Nix client with the current daemon.
 
   **Expected value**: something like `/nix/store/...-nix-2.18.2`
-- `NIX_TESTS_CA_BY_DEFAULT` - Pass `__contentAddressed`, `outputHashMode` and `outputHashAlgo` to builds of some input-addressed derivations in the test suite.
-
-  **Expected value**: 1
 - `TEST_DATA` - Not an environment variable! This is used in repl characterization tests to refer to `tests/functional/repl_characterization/data`.
   More specifically, that path is replaced with the string `$TEST_DATA` in output for reproducibility.
 - `TEST_HOME` (output) - Set to the temporary directory that is set as `$HOME` inside the tests, underneath `$TEST_ROOT`.

doc/manual/src/glossary.md

@@ -41,12 +41,6 @@
 
   [realise]: #gloss-realise
 
-- [content-addressed derivation]{#gloss-content-addressed-derivation}
-
-  A derivation which has the
-  [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed)
-  attribute set to `true`.
-
 - [fixed-output derivation]{#gloss-fixed-output-derivation}
 
   A derivation which includes the
@@ -114,14 +108,13 @@
 - [input-addressed store object]{#gloss-input-addressed-store-object}
 
   A store object produced by building a
-  non-[content-addressed](#gloss-content-addressed-derivation),
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
 - [output-addressed store object]{#gloss-output-addressed-store-object}
 
   A [store object] whose [store path] is determined by its contents.
-  This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
+  This includes derivations and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
 - [substitute]{#gloss-substitute}
 

doc/manual/src/installation/prerequisites-source.md

@@ -54,11 +54,6 @@ The most current alternative to this section is to read `package.nix` and see wh
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
-  - The `libsodium` library for verifying cryptographic signatures
-    of contents fetched from binary caches.
-    It can be obtained from the official web site
-    <https://libsodium.org>.
-
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from

doc/manual/src/language/advanced-attributes.md

@@ -209,15 +209,8 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
-    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
-    >
-    > To use this attribute, you must enable the
-    > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
-    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
-    >
-    > ```
-    > extra-experimental-features = ca-derivations
-    > ```
+    > This attribute is part of a removed [experimental feature](@docroot@/contributing/experimental-features.md).
+    > Setting this flag *will* cause eval errors.
 
     If this attribute is set to `true`, then the derivation
     outputs will be stored in a content-addressed location rather than the
@@ -309,8 +302,6 @@ Derivations can declare some infrequently used optional attributes.
 
     - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object).
     - `maxClosureSize` defines the maximum size of the output's closure.
-    - `ignoreSelfRefs` controls whether self-references should be considered when
-      checking for allowed references/requisites.
 
     Example:
 

doc/manual/src/package-management/garbage-collection.md

@@ -71,3 +71,62 @@ $ nix-collect-garbage -d
 ```
 
 is a quick and easy way to clean up your system.
+
+## Garbage Collector Roots
+
+### Explicit roots
+
+All store paths to which there are symlinks in the directory
+`prefix/nix/var/nix/gcroots` will be used as roots by the garbage
+collector. For instance, the following command makes the path
+`/nix/store/d718ef...-foo` a root of the collector:
+
+```console
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+```
+
+That is, after this command, the garbage collector will not remove
+`/nix/store/d718ef...-foo` or any of its dependencies.
+
+Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
+symlinks.
+
+Symlinks may also point to paths outside the nix store. If the
+destination of the symlink is itself a symlink to a store path, it
+is also considered a root. This style of GC root is called an
+"indirect root", and is created by tools like `nix-build` to avoid
+garbage-collecting paths that are being used on-the-fly rather than
+installed in profiles.
+
+
+### In-use roots
+
+Lix will also perform a best-effort detection of paths that are in use
+by running processes when scanning for garbage collection roots, to
+avoid removing paths that are still needed by running processes.
+
+Exact details vary between platforms, but the following will generally
+be taken into account:
+
+- Executables in the store that are currently running;
+- Other files in the store that are mapped into a process's address space (e.g. shared libraries);
+- Files in the store to which processes have open handles;
+- Store paths found in processes' environment variables.
+
+Note that this detection is susceptible to missing paths that may still be in use for multiple reasons:
+
+- Time-of-check-to-time-of-use (TOCTTOU): new processes may appear
+  after Lix has enumerated the currently running processes, and will
+  not be taken into account;
+- Access privileges: if the garbage collection is not running as the
+  root user (this is typically the case for single-user
+  installations), it will not be able to scan processes belonging to
+  other users;
+- Other types of references: store paths may be stored in parts of the
+  filesystem (e.g. databases) or process memory (e.g. environment
+  variables changed since the start of the process) that Lix does not
+  scan.
+
+For this reason, it is recommended to create explicit roots whenever
+using store paths that aren't obtained from some existing explicit GC
+root.

doc/manual/src/package-management/garbage-collector-roots.md

@@ -1,18 +0,0 @@
-# Garbage Collector Roots
-
-The roots of the garbage collector are all store paths to which there
-are symlinks in the directory `prefix/nix/var/nix/gcroots`. For
-instance, the following command makes the path
-`/nix/store/d718ef...-foo` a root of the collector:
-
-```console
-$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
-```
-
-That is, after this command, the garbage collector will not remove
-`/nix/store/d718ef...-foo` or any of its dependencies.
-
-Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
-symlinks. Symlinks to non-store paths are followed and searched for
-roots, but symlinks to non-store paths *inside* the paths reached in
-that way are not followed to prevent infinite recursion.

doc/manual/src/protocols/derivation-aterm.md

@@ -2,18 +2,8 @@
 
 For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
 
-Derivations are serialised in one of the following formats:
+Derivations are serialised in the following format:
 
-- ```
-  Derive(...)
-  ```
-
-  For all stable derivations.
-
-- ```
-  DrvWithVersion(<version-string>, ...)
-  ```
-
-  The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
-
-  - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.
+```
+Derive(...)
+```

doc/manual/substitute.py

@@ -90,6 +90,11 @@ def recursive_replace(data, book_root, search_path):
                     ).replace(
                         '@docroot@',
                         ("../" * len(path_to_chapter.parent.parts) or "./")[:-1]
+                    ).replace(
+                        # this replacement is to avoid corrupting the
+                        # hacking.md manual section on docroot
+                        '@\\docroot\\@',
+                        '@docroot@',
                     ),
                     sub_items = [
                         recursive_replace(sub_item, book_root, search_path)

docker.nix

@@ -62,38 +62,37 @@ let
     ++ autoLayered
     ++ extraPkgs;
 
-  users =
-    {
+  users = {
 
-      root = {
-        uid = 0;
-        shell = "${pkgs.bashInteractive}/bin/bash";
-        home = "/root";
-        gid = 0;
-        groups = [ "root" ];
-        description = "System administrator";
-      };
+    root = {
+      uid = 0;
+      shell = "${pkgs.bashInteractive}/bin/bash";
+      home = "/root";
+      gid = 0;
+      groups = [ "root" ];
+      description = "System administrator";
+    };
 
-      nobody = {
-        uid = 65534;
-        shell = "${pkgs.shadow}/bin/nologin";
-        home = "/var/empty";
-        gid = 65534;
-        groups = [ "nobody" ];
-        description = "Unprivileged account (don't use!)";
+    nobody = {
+      uid = 65534;
+      shell = "${pkgs.shadow}/bin/nologin";
+      home = "/var/empty";
+      gid = 65534;
+      groups = [ "nobody" ];
+      description = "Unprivileged account (don't use!)";
+    };
+  }
+  // lib.listToAttrs (
+    map (n: {
+      name = "nixbld${toString n}";
+      value = {
+        uid = 30000 + n;
+        gid = 30000;
+        groups = [ "nixbld" ];
+        description = "Nix build user ${toString n}";
       };
-    }
-    // lib.listToAttrs (
-      map (n: {
-        name = "nixbld${toString n}";
-        value = {
-          uid = 30000 + n;
-          gid = 30000;
-          groups = [ "nixbld" ];
-          description = "Nix build user ${toString n}";
-        };
-      }) (lib.lists.range 1 32)
-    );
+    }) (lib.lists.range 1 32)
+  );
 
   groups = {
     root.gid = 0;
@@ -193,13 +192,11 @@ let
             in
             ''
               {
-                ${
-                  lib.concatStringsSep "\n" (
-                    builtins.map (output: ''
-                      ${output} = { outPath = "${lib.getOutput output drv}"; };
-                    '') outputs
-                  )
-                }
+                ${lib.concatStringsSep "\n" (
+                  builtins.map (output: ''
+                    ${output} = { outPath = "${lib.getOutput output drv}"; };
+                  '') outputs
+                )}
                 outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
                 name = "${drv.name}";
                 outPath = "${drv}";
@@ -361,8 +358,10 @@ let
         "org.opencontainers.image.source" = "https://git.lix.systems/lix-project/lix";
         "org.opencontainers.image.vendor" = "Lix project";
         "org.opencontainers.image.version" = pkgs.nix.version;
-        "org.opencontainers.image.description" = "Minimal Lix container image, with some batteries included.";
-      } // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
+        "org.opencontainers.image.description" =
+          "Minimal Lix container image, with some batteries included.";
+      }
+      // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
     };
 
     meta = {

flake.lock

@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
@@ -32,18 +48,44 @@
         "type": "github"
       }
     },
-    "nixpkgs": {
+    "nix_2_18": {
+      "inputs": {
+        "flake-compat": [
+          "flake-compat"
+        ],
+        "lowdown-src": "lowdown-src",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": [
+          "nixpkgs-regression"
+        ]
+      },
       "locked": {
-        "lastModified": 1733348545,
-        "narHash": "sha256-b4JrUmqT0vFNx42aEN9LTWOHomkTKL/ayLopflVf81U=",
+        "lastModified": 1730375271,
+        "narHash": "sha256-RrOFlDGmRXcVRV2p2HqHGqvzGNyWoD0Dado/BNlJ1SI=",
         "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "9ecb50d2fae8680be74c08bb0a995c5383747f89",
+        "repo": "nix",
+        "rev": "0f665ff6779454f2117dcc32e44380cda7f45523",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11-small",
+        "ref": "2.18.9",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1705033721,
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -64,6 +106,22 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1758391731,
+        "narHash": "sha256-UuwQoPWv13DVKMveeev+F0OC/N95AOmAz6SzCuGhxjQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "3f00d36f15e16e0471d9ca1e8f88958941fa970a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-25.05-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
@@ -84,7 +142,8 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "nix2container": "nix2container",
-        "nixpkgs": "nixpkgs",
+        "nix_2_18": "nix_2_18",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-regression": "nixpkgs-regression",
         "pre-commit-hooks": "pre-commit-hooks"
       }

flake.nix

@@ -2,8 +2,19 @@
   description = "Lix: A modern, delicious implementation of the Nix package manager";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
     nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
+
+    # Required because Nix 2.18 is not in Nixpkgs ≥ 25.05 anymore.
+    nix_2_18 = {
+      url = "github:NixOS/nix/2.18.9";
+      # NOTE(Raito): this is not possible because patches on libseccomp does not apply anymore on this Nix.
+      # Let's keep the latest known nixpkgs useable with Nix 2.18 for our tests.
+      # inputs.nixpkgs.follows = "nixpkgs";
+      inputs.nixpkgs-regression.follows = "nixpkgs-regression";
+      inputs.flake-compat.follows = "flake-compat";
+    };
+
     pre-commit-hooks = {
       url = "github:cachix/git-hooks.nix";
       flake = false;
@@ -25,6 +36,7 @@
       nixpkgs-regression,
       pre-commit-hooks,
       nix2container,
+      nix_2_18,
       flake-compat,
     }:
 
@@ -163,6 +175,16 @@
         {
           nixStable = prev.nix;
 
+          nixVersions = prev.nixVersions // {
+            nix_2_3 = prev.nixVersions.nix_2_3.overrideAttrs (old: {
+              meta = old.meta // {
+                knownVulnerabilities = [ ];
+              };
+            });
+            # Nix 2.18 has been removed from Nixpkgs ≥ 25.05, so we need to reintroduce it ourselves for our tests.
+            nix_2_18 = nix_2_18.outputs.packages.${currentStdenv.hostPlatform.system}.default;
+          };
+
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
@@ -221,6 +243,22 @@
           boehmgc-nix = final.nix.passthru.boehmgc-nix;
           # And same thing for our build-release-notes package.
           build-release-notes = final.nix.passthru.build-release-notes;
+
+          lowdown_1_3 =
+            # If the stable channel we are using ships lowdown >= 1.4, we need
+            # to swap this around, take the default lowdown from the stable
+            # channel and add an overridden one for the legacy version.
+            assert lib.versionOlder prev.lowdown.version "1.4.0";
+            prev.lowdown;
+          lowdown = prev.lowdown.overrideAttrs (prevAttrs: rec {
+            version = "2.0.2";
+            src = final.fetchurl {
+              url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz";
+              sha512 = "2a4d0rqh8gkw4ca3gkzddp0hjpmmw74cbks8k0inhh0vizmgbn188zdv6m1kgmr019b99g7insli8js3ci1ji7y4n5nk704bswf3z3i";
+            };
+            nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ final.buildPackages.bmake ];
+            postInstall = lib.replaceStrings [ "lowdown.so.1" ] [ "lowdown.so.2" ] prevAttrs.postInstall;
+          });
         };
     in
     {
@@ -235,6 +273,30 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
+        # Building Lix twice in CI is expensive, but we can catch a lot of static
+        # build regressions by at least making sure it evals and configures.
+        configure-static = lib.genAttrs linux64BitSystems (
+          system:
+          self.packages.${system}.nix-static.overrideAttrs {
+            dontBuild = true;
+            installPhase = ''
+              runHook preInstall
+
+              echo "configure-static complete. exiting with success"
+              mkdir -p "$out"
+              exit 0
+            '';
+          }
+        );
+
+        # Ensure support for lowdown < 1.4 doesn't regress
+        build-lowdown_1_3 = forAllSystems (
+          system:
+          self.packages.${system}.nix.override {
+            lowdown = nixpkgsFor.${system}.native.lowdown_1_3;
+          }
+        );
+
         devShell = forAllSystems (system: {
           default = self.devShells.${system}.default;
           clang = self.devShells.${system}.native-clangStdenvPackages;
@@ -381,13 +443,16 @@
               in
               pkgs.symlinkJoin {
                 name = "nixpkgs-lib-tests";
-                paths =
-                  [ testWithNix ]
-                  # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override
-                  # the evaluator used for the test. it will break again in the future, don't worry.
-                  ++ lib.optionals pkgs.stdenv.isLinux [
-                    (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset
-                  ];
+                paths = [
+                  testWithNix
+                ]
+                # NOTE: nixpkgs 25.05 is being ... *creative*, and requires this dance to override
+                # the evaluator used for the test. it will break again in the future, don't worry.
+                ++ lib.optionals pkgs.stdenv.isLinux [
+                  ((pkgs.callPackage "${nixpkgs}/ci/eval" { inherit nix; }).attrpathsSuperset {
+                    evalSystem = system;
+                  })
+                ];
               }
             );
           };
@@ -436,6 +501,7 @@
           # devShells and packages already get checked by nix flake check, so
           # this is just jobs that are special
 
+          build-lowdown_1_3 = self.hydraJobs.build-lowdown_1_3.${system};
           binaryTarball = self.hydraJobs.binaryTarball.${system};
           perlBindings = self.hydraJobs.perlBindings.${system};
           nix-eval-jobs = self.hydraJobs.nix-eval-jobs.${system};
@@ -460,7 +526,10 @@
         }
         // (
           lib.optionalAttrs (builtins.elem system linux64BitSystems) {
-            nix-static = nixpkgsFor.${system}.static.nix;
+            # python doesn't work in static builds as of 2025-06-27
+            nix-static = nixpkgsFor.${system}.static.nix.overrideAttrs (_: {
+              doCheck = false;
+            });
             dockerImage =
               let
                 pkgs = nixpkgsFor.${system}.native;

justfile

@@ -41,6 +41,10 @@ test-unit *OPTIONS: (test "--suite" "check")
 # Run integration tests only
 test-integration *OPTIONS: install (test "--suite" "installcheck")
 
+# Run functional2 tests using pytest directly, allowing for additional arguments to be passed to pytest e.g. for more granular test selection
+test-functional2 *OPTIONS:
+    cd tests && python -m pytest -v {{ OPTIONS }} functional2
+
 alias clang-tidy := lint
 
 # Lint with `clang-tidy`

lix/OWNERS

@@ -0,0 +1,10 @@
+# This reproduces the Lix Approvers plus Lix groups to yield the status quo
+alois1@gmx-topmail.de
+jade@lix.systems
+lunaphied@lunaphied.me
+maximilian@mbosch.me
+me@0upti.me
+pennae@lix.systems
+qyriad@qyriad.me
+raito@lix.systems
+rbt@sent.as

lix/code-generation/capnpc_wrapper.py

@@ -0,0 +1,64 @@
+#!@python@
+
+import argparse
+import capnp
+from pathlib import Path
+import os
+import subprocess
+import sys
+
+if lang := os.environ.get('lix_capnp_lang'):
+    outputs = os.environ['lix_capnp_outputs'].split()
+    old_cwd = os.environ['lix_capnp_old_cwd']
+    schema = capnp.load('@capnp_include@/capnp/schema.capnp', imports=['@capnp_include@'])
+    request = schema.CodeGeneratorRequest.read(sys.stdin)
+
+    subprocess.run([lang], input=request.as_builder().to_bytes()).check_returncode()
+
+    base_dir = os.getcwd()
+    os.chdir(old_cwd)
+
+    include = [ str(Path(p).resolve()) for p in os.environ['lix_capnp_include'].split(':') ]
+
+    if depfile := os.environ['lix_capnp_depfile']:
+        deps = ""
+        for input in request.requestedFiles:
+            deps += " ".join(f"{input.filename}.{o}" for o in outputs)
+            deps += ":"
+            for dep in input.imports:
+                if dep.name.startswith("/"):
+                    for candidate in (Path(i + dep.name) for i in include):
+                        if candidate.exists():
+                            deps += " " + str(candidate)
+                            break
+                else:
+                    raise RuntimeError("not handling relative includes")
+            deps += "\n\n"
+        Path(depfile).write_text(deps)
+else:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--language')
+    parser.add_argument('--outdir')
+    parser.add_argument('--src-prefix')
+    parser.add_argument('--depfile', default="")
+    parser.add_argument('-I', '--include', action='append', default=['@capnp_include@'])
+    parser.add_argument('inputs', nargs='+')
+    args = parser.parse_args()
+
+    for infile in args.inputs:
+        os.environ['lix_capnp_lang'] = f"capnpc-{args.language}"
+        os.environ['lix_capnp_include'] = ':'.join(args.include)
+        os.environ['lix_capnp_depfile'] = args.depfile
+        os.environ['lix_capnp_old_cwd'] = os.getcwd()
+        if args.language == "c++":
+            os.environ['lix_capnp_outputs'] = "c++ h"
+        else:
+            raise RuntimeError("unknown language " + args.language)
+        subprocess.run([
+            '@capnp@',
+            'compile',
+            f'-o{sys.argv[0]}:{args.outdir}',
+            f'--src-prefix={args.src_prefix}',
+            *(f"-I{i}" for i in args.include),
+            infile
+        ]).check_returncode()

lix/legacy/build-remote.cc

@@ -1,7 +1,19 @@
+#include "lix/libstore/path.hh"
+#include "lix/libutil/async.hh"
+#include "lix/libutil/error.hh"
+#include "lix/libutil/file-descriptor.hh"
+#include "lix/libutil/logging.hh"
+#include "lix/libutil/rpc.hh"
+#include "lix/libutil/types-rpc.hh"
 #include <algorithm>
+#include <capnp/rpc-twoparty.h>
 #include <chrono>
+#include <cstring>
+#include <future>
+#include <kj/time.h>
 #include <set>
 #include <memory>
+#include <string>
 #include <tuple>
 #if __APPLE__
 #include <sys/time.h>
@@ -17,13 +29,27 @@
 #include "lix/libstore/derivations.hh"
 #include "lix/libutil/strings.hh"
 #include "lix/libstore/local-store.hh"
+#include "lix/libstore/types-rpc.hh"
 #include "lix/libcmd/legacy.hh"
 #include "lix/libutil/experimental-features.hh"
 #include "lix/libutil/hash.hh"
 #include "build-remote.hh"
 
+#include "lix/libstore/build/hook-instance.capnp.h"
+
 namespace nix {
 
+namespace {
+struct Instance final : rpc::build_remote::HookInstance::Server
+{
+    unsigned int maxBuildJobs;
+
+    Instance(unsigned int maxBuildJobs) : maxBuildJobs(maxBuildJobs) {}
+
+    kj::Promise<void> build(BuildContext context) override;
+};
+}
+
 std::string escapeUri(std::string uri)
 {
     std::replace(uri.begin(), uri.end(), '/', '_');
@@ -53,6 +79,245 @@ static bool allSupportedLocally(Store & store, const std::set<std::string>& requ
     return true;
 }
 
+static std::tuple<bool, Machine *, AutoCloseFD> selectBestMachine(
+    Machines & machines,
+    const std::string & neededSystem,
+    const std::set<std::string> & requiredFeatures
+)
+{
+    bool rightType = false;
+    Machine * bestMachine = nullptr;
+    AutoCloseFD bestSlotLock;
+    uint64_t bestLoad = 0;
+
+    for (auto & m : machines) {
+        debug("considering building on remote machine '%s'", m.storeUri);
+
+        if (m.enabled && m.systemSupported(neededSystem) && m.allSupported(requiredFeatures)
+            && m.mandatoryMet(requiredFeatures))
+        {
+            rightType = true;
+            AutoCloseFD free;
+            uint64_t load = 0;
+            for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
+                auto slotLock = openSlotLock(m, slot);
+                if (tryLockFile(slotLock.get(), ltWrite)) {
+                    if (!free) {
+                        free = std::move(slotLock);
+                    }
+                } else {
+                    ++load;
+                }
+            }
+            if (!free) {
+                continue;
+            }
+            bool best = false;
+            if (!bestSlotLock) {
+                best = true;
+            } else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) {
+                best = true;
+            } else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) {
+                if (m.speedFactor > bestMachine->speedFactor) {
+                    best = true;
+                } else if (m.speedFactor == bestMachine->speedFactor) {
+                    if (load < bestLoad) {
+                        best = true;
+                    }
+                }
+            }
+            if (best) {
+                bestLoad = load;
+                bestSlotLock = std::move(free);
+                bestMachine = &m;
+            }
+        }
+    }
+
+    return {rightType, bestMachine, std::move(bestSlotLock)};
+}
+
+static void printSelectionFailureMessage(
+    Verbosity level,
+    const std::string_view drvstr,
+    const Machines & machines,
+    const std::string & neededSystem,
+    const std::set<std::string> & requiredFeatures
+)
+{
+    std::string machinesFormatted;
+
+    for (auto & m : machines) {
+        machinesFormatted += HintFmt(
+                                 "\n([%s], %s, [%s], [%s])",
+                                 concatStringsSep<StringSet>(", ", m.systemTypes),
+                                 m.maxJobs,
+                                 concatStringsSep<StringSet>(", ", m.supportedFeatures),
+                                 concatStringsSep<StringSet>(", ", m.mandatoryFeatures)
+        )
+                                 .str();
+    }
+
+    printMsg(
+        level,
+        "Failed to find a machine for remote build!\n"
+        "derivation: %s\n"
+        "required (system, features): (%s, [%s])\n"
+        "%s available machines:\n"
+        "(systems, maxjobs, supportedFeatures, mandatoryFeatures)%s",
+        drvstr,
+        neededSystem,
+        concatStringsSep<StringSet>(", ", requiredFeatures),
+        machines.size(),
+        Uncolored(machinesFormatted)
+    );
+}
+
+namespace {
+struct BuilderConnection
+{
+    AutoCloseFD slotLock;
+    std::shared_ptr<Store> sshStore;
+    std::string storeUri;
+    Pipe logPipe;
+
+    // start the thread that reads ssh stderr and turns it into log items.
+    // this future *must* outlive sshStore, otherwise it will never finish
+    std::future<void> startLogThread(int intoFD)
+    {
+        if (!logPipe.readSide) {
+            return {};
+        }
+
+        logPipe.writeSide.close();
+
+        return std::async(
+            std::launch::async,
+            [](int from, int to) {
+                AsyncIoRoot aio;
+
+                auto reader = AIO().lowLevelProvider.wrapInputFd(from);
+                auto writer = AIO().lowLevelProvider.wrapOutputFd(to);
+
+                reader->pumpTo(*writer).wait(aio.kj.waitScope);
+            },
+            logPipe.readSide.get(),
+            intoFD
+        );
+    }
+};
+
+struct AcceptedBuild final : rpc::build_remote::HookInstance::AcceptedBuild::Server
+{
+    ref<Store> store;
+    StorePath drvPath;
+    BuilderConnection builder;
+    rpc::build_remote::HookInstance::BuildLogger::Client buildLogger;
+
+    AcceptedBuild(
+        ref<Store> store,
+        StorePath drvPath,
+        BuilderConnection builder,
+        rpc::build_remote::HookInstance::BuildLogger::Client buildLogger
+    )
+        : store(store)
+        , drvPath(drvPath)
+        , builder(std::move(builder))
+        , buildLogger(std::move(buildLogger))
+    {
+    }
+
+    kj::Promise<void> run(RunContext context) override;
+};
+
+enum class BuildRejected { Temporarily, Permanently };
+}
+
+static kj::Promise<Result<std::variant<BuildRejected, BuilderConnection>>> connectToBuilder(
+    const ref<Store> & store,
+    const std::optional<StorePath> & drvPath,
+    Machines & machines,
+    const unsigned int maxBuildJobs,
+    const bool amWilling,
+    const std::string & neededSystem,
+    const std::set<std::string> & requiredFeatures
+)
+try {
+    AutoCloseFD bestSlotLock;
+
+    /* It would be possible to build locally after some builds clear out,
+       so don't show the warning now: */
+    bool couldBuildLocally = maxBuildJobs > 0
+        && (neededSystem == settings.thisSystem
+            || settings.extraPlatforms.get().count(neededSystem) > 0)
+        && allSupportedLocally(*store, requiredFeatures);
+    /* It's possible to build this locally right now: */
+    bool canBuildLocally = amWilling && couldBuildLocally;
+
+    /* Error ignored here, will be caught later */
+    mkdir(currentLoad.c_str(), 0777);
+
+    while (true) {
+        bestSlotLock.reset();
+        AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true);
+        TRY_AWAIT(lockFileAsync(lock.get(), ltWrite));
+
+        auto [rightType, bestMachine, slotLock] =
+            selectBestMachine(machines, neededSystem, requiredFeatures);
+        bestSlotLock = std::move(slotLock);
+
+        if (!bestSlotLock) {
+            if (rightType && !canBuildLocally) {
+                co_return BuildRejected::Temporarily;
+            } else {
+                printSelectionFailureMessage(
+                    couldBuildLocally ? lvlChatty : lvlWarn,
+                    drvPath ? drvPath->to_string() : "<unknown>",
+                    machines,
+                    neededSystem,
+                    requiredFeatures
+                );
+
+                co_return BuildRejected::Permanently;
+            }
+        }
+
+#if __APPLE__
+        futimes(bestSlotLock.get(), nullptr);
+#else
+        futimens(bestSlotLock.get(), nullptr);
+#endif
+
+        lock.reset();
+
+        std::shared_ptr<Store> sshStore;
+        Pipe logPipe;
+
+        try {
+            Activity act(
+                *logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)
+            );
+
+            std::tie(sshStore, logPipe) = TRY_AWAIT(bestMachine->openStore());
+            TRY_AWAIT(sshStore->connect());
+            co_return BuilderConnection{
+                std::move(bestSlotLock), sshStore, bestMachine->storeUri, std::move(logPipe)
+            };
+        } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
+            std::string msg = logPipe.readSide ? chomp(drainFD(logPipe.readSide.get(), false)) : "";
+            printError(
+                "cannot build on '%s': %s%s",
+                bestMachine->storeUri,
+                e.what(),
+                msg.empty() ? "" : ": " + msg
+            );
+            bestMachine->enabled = false;
+        }
+    }
+} catch (...) {
+    co_return result::current_exception();
+}
+
 static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings argv)
 {
     {
@@ -73,7 +338,7 @@ static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings
         FdSource source(STDIN_FILENO);
 
         /* Read the parent's settings. */
-        while (readInt(source)) {
+        while (readNum<unsigned>(source)) {
             auto name = readString(source);
             auto value = readString(source);
             settings.set(name, value);
@@ -84,7 +349,23 @@ static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings
 
         initPlugins();
 
-        auto store = aio.blockOn(openStore());
+        auto conn = aio.kj.lowLevelProvider->wrapUnixSocketFd(1);
+        capnp::TwoPartyServer srv(kj::heap<Instance>(maxBuildJobs));
+        srv.accept(*conn, 1).wait(aio.kj.waitScope);
+        return 0;
+    }
+}
+
+kj::Promise<void> Instance::build(BuildContext context)
+{
+    try {
+        // FIXME this does not open a daemon connection for historical reasons.
+        // we may create a lot of build hook instances, and having each of them
+        // also create a daemon instance is inefficient and wasteful. in future
+        // versions of the build hook (where we don't need one hook process per
+        // build) we should change this to using a daemon connection, ideally a
+        // daemon connection provided by the parent via file descriptor passing
+        auto store = TRY_AWAIT(openStore(settings.storeUri, {}, AllowDaemon::Disallow));
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
@@ -94,181 +375,69 @@ static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings
         else
             currentLoad = settings.nixStateDir + currentLoadName;
 
-        std::shared_ptr<Store> sshStore;
-        AutoCloseFD bestSlotLock;
-
         auto machines = getMachines();
         debug("got %d remote builders", machines.size());
 
         if (machines.empty()) {
-            std::cerr << "# decline-permanently\n";
-            return 0;
+            context.getResults().initResult().initGood().setDeclinePermanently();
+            co_return;
         }
 
-        std::optional<StorePath> drvPath;
-        std::string storeUri;
+        auto amWilling = context.getParams().getAmWilling();
+        auto neededSystem = rpc::to<std::string>(context.getParams().getNeededSystem());
+        auto drvPath = from(context.getParams().getDrvPath(), *store);
+        auto requiredFeatures =
+            rpc::to<std::set<std::string>>(context.getParams().getRequiredFeatures());
+        auto buildLogger = context.getParams().getBuildLogger();
 
-        while (true) {
+        auto result = TRY_AWAIT(connectToBuilder(
+            store, drvPath, machines, maxBuildJobs, amWilling, neededSystem, requiredFeatures
+        ));
 
-            try {
-                auto s = readString(source);
-                if (s != "try") return 0;
-            } catch (EndOfFile &) { return 0; }
-
-            auto amWilling = readInt(source);
-            auto neededSystem = readString(source);
-            drvPath = store->parseStorePath(readString(source));
-            auto requiredFeatures = readStrings<std::set<std::string>>(source);
-
-            /* It would be possible to build locally after some builds clear out,
-               so don't show the warning now: */
-            bool couldBuildLocally = maxBuildJobs > 0
-                 &&  (  neededSystem == settings.thisSystem
-                     || settings.extraPlatforms.get().count(neededSystem) > 0)
-                 &&  allSupportedLocally(*store, requiredFeatures);
-            /* It's possible to build this locally right now: */
-            bool canBuildLocally = amWilling && couldBuildLocally;
-
-            /* Error ignored here, will be caught later */
-            mkdir(currentLoad.c_str(), 0777);
-
-            while (true) {
-                bestSlotLock.reset();
-                AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true);
-                lockFile(lock.get(), ltWrite);
-
-                bool rightType = false;
-
-                Machine * bestMachine = nullptr;
-                uint64_t bestLoad = 0;
-                for (auto & m : machines) {
-                    debug("considering building on remote machine '%s'", m.storeUri);
-
-                    if (m.enabled &&
-                        m.systemSupported(neededSystem) &&
-                        m.allSupported(requiredFeatures) &&
-                        m.mandatoryMet(requiredFeatures))
-                    {
-                        rightType = true;
-                        AutoCloseFD free;
-                        uint64_t load = 0;
-                        for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
-                            auto slotLock = openSlotLock(m, slot);
-                            if (tryLockFile(slotLock.get(), ltWrite)) {
-                                if (!free) {
-                                    free = std::move(slotLock);
-                                }
-                            } else {
-                                ++load;
-                            }
-                        }
-                        if (!free) {
-                            continue;
-                        }
-                        bool best = false;
-                        if (!bestSlotLock) {
-                            best = true;
-                        } else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) {
-                            best = true;
-                        } else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) {
-                            if (m.speedFactor > bestMachine->speedFactor) {
-                                best = true;
-                            } else if (m.speedFactor == bestMachine->speedFactor) {
-                                if (load < bestLoad) {
-                                    best = true;
-                                }
-                            }
-                        }
-                        if (best) {
-                            bestLoad = load;
-                            bestSlotLock = std::move(free);
-                            bestMachine = &m;
-                        }
-                    }
-                }
-
-                if (!bestSlotLock) {
-                    if (rightType && !canBuildLocally)
-                        std::cerr << "# postpone\n";
-                    else
-                    {
-                        // add the template values.
-                        std::string drvstr;
-                        if (drvPath.has_value())
-                            drvstr = drvPath->to_string();
-                        else
-                            drvstr = "<unknown>";
-
-                        std::string machinesFormatted;
-
-                        for (auto & m : machines) {
-                            machinesFormatted += HintFmt(
-                                "\n([%s], %s, [%s], [%s])",
-                                concatStringsSep<StringSet>(", ", m.systemTypes),
-                                m.maxJobs,
-                                concatStringsSep<StringSet>(", ", m.supportedFeatures),
-                                concatStringsSep<StringSet>(", ", m.mandatoryFeatures)
-                            ).str();
-                        }
-
-                        auto error = HintFmt(
-                            "Failed to find a machine for remote build!\n"
-                            "derivation: %s\n"
-                            "required (system, features): (%s, [%s])\n"
-                            "%s available machines:\n"
-                            "(systems, maxjobs, supportedFeatures, mandatoryFeatures)%s",
-                            drvstr,
-                            neededSystem,
-                            concatStringsSep<StringSet>(", ", requiredFeatures),
-                            machines.size(),
-                            Uncolored(machinesFormatted)
-                       );
-
-                        printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str());
-
-                        std::cerr << "# decline\n";
-                    }
-                    break;
-                }
-
-#if __APPLE__
-                futimes(bestSlotLock.get(), nullptr);
-#else
-                futimens(bestSlotLock.get(), nullptr);
-#endif
-
-                lock.reset();
-
-                try {
-
-                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
-
-                    sshStore = aio.blockOn(bestMachine->openStore());
-                    aio.blockOn(sshStore->connect());
-                    storeUri = bestMachine->storeUri;
-
-                } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
-                    auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        msg.empty() ? "" : ": " + msg);
-                    bestMachine->enabled = false;
-                    continue;
-                }
-
-                goto connected;
+        if (auto immediateResponse = std::get_if<BuildRejected>(&result)) {
+            switch (*immediateResponse) {
+            case BuildRejected::Temporarily:
+                context.getResults().initResult().initGood().setPostpone();
+                co_return;
+            case BuildRejected::Permanently:
+                context.getResults().initResult().initGood().setDecline();
+                co_return;
             }
         }
 
-connected:
-        close(5);
+        auto builder = std::get_if<BuilderConnection>(&result);
+        assert(builder);
 
-        assert(sshStore);
+        auto ac = context.getResults().initResult().initGood().initAccept();
+        RPC_FILL(ac, setMachineName, builder->storeUri);
+        ac.setMachine(kj::heap<AcceptedBuild>(store, drvPath, std::move(*builder), buildLogger));
+    } catch (...) {
+        RPC_FILL(context.getResults(), initResult, std::current_exception());
+    }
+}
 
-        std::cerr << "# accept\n" << storeUri << "\n";
+kj::Promise<void> AcceptedBuild::run(RunContext context)
+{
+    try {
+        const int logFD = (co_await buildLogger.getFd()).orDefault(-1);
+        if (logFD < 0) {
+            throw Error("build-hook needs a logFD from the builder to build");
+        }
 
-        auto inputs = readStrings<PathSet>(source);
-        auto wantedOutputs = readStrings<StringSet>(source);
+        auto logThread = builder.startLogThread(logFD);
+        KJ_DEFER({
+            // drop any existing ssh connection so the log thread can exit
+            builder.sshStore = nullptr;
+            if (logThread.valid()) {
+                logThread.get();
+            }
+        });
+
+        auto & sshStore = builder.sshStore;
+        auto & storeUri = builder.storeUri;
+
+        auto inputs = rpc::to<std::set<StorePath>>(context.getParams().getInputs(), *store);
+        auto wantedOutputs = rpc::to<std::set<std::string>>(context.getParams().getWantedOutputs());
 
         auto lockFileName = currentLoad + "/" + makeLockFilename(storeUri) + ".upload-lock";
 
@@ -277,27 +446,24 @@ connected:
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
 
-            if (!unsafeLockFileSingleThreaded(uploadLock.get(), ltWrite, std::chrono::minutes(15)))
+            auto result = TRY_AWAIT(
+                AIO().timeoutAfter(15 * kj::MINUTES, lockFileAsync(uploadLock.get(), ltWrite))
+            );
+            if (!result) {
                 printError("somebody is hogging the upload lock for '%s', continuing...");
+            }
         }
 
         auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            aio.blockOn(copyPaths(
-                *store,
-                *sshStore,
-                store->parseStorePathSet(inputs),
-                NoRepair,
-                NoCheckSigs,
-                substitute
-            ));
+            TRY_AWAIT(copyPaths(*store, *sshStore, inputs, NoRepair, NoCheckSigs, substitute));
         }
 
         uploadLock.reset();
 
-        auto drv = aio.blockOn(store->readDerivation(*drvPath));
+        auto drv = TRY_AWAIT(store->readDerivation(drvPath));
 
         std::optional<BuildResult> optResult;
 
@@ -305,7 +471,7 @@ connected:
         // stores), we assume we are. This is necessary for backwards
         // compat.
         bool trustedOrLegacy = ({
-            std::optional trusted = aio.blockOn(sshStore->isTrustedClient());
+            std::optional trusted = TRY_AWAIT(sshStore->isTrustedClient());
             !trusted || *trusted;
         });
 
@@ -324,52 +490,39 @@ connected:
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.map.empty())
-                drv.inputSrcs = store->parseStorePathSet(inputs);
-            optResult = aio.blockOn(sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv));
-            auto & result = *optResult;
-            if (!result.success())
-                throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
+            if (!drv.inputDrvs.empty()) {
+                drv.inputSrcs = inputs;
+            }
+            optResult =
+                TRY_AWAIT(sshStore->buildDerivation(drvPath, (const BasicDerivation &) drv));
         } else {
-            aio.blockOn(copyClosure(
-                *store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute
+            TRY_AWAIT(copyClosure(
+                *store, *sshStore, StorePathSet{drvPath}, NoRepair, NoCheckSigs, substitute
             ));
-            auto res = aio.blockOn(sshStore->buildPathsWithResults({
-                DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(*drvPath),
-                    .outputs = OutputsSpec::All {},
-                }
-            }));
+            auto res = TRY_AWAIT(sshStore->buildPathsWithResults({DerivedPath::Built{
+                .drvPath = makeConstantStorePath(drvPath),
+                .outputs = OutputsSpec::All{},
+            }}));
             // One path to build should produce exactly one build result
             assert(res.size() == 1);
             optResult = std::move(res[0]);
         }
 
+        auto & result = *optResult;
+        if (!result.success()) {
+            throw Error(
+                "build of '%s' on '%s' failed: %s",
+                store->printStorePath(drvPath),
+                storeUri,
+                result.errorMsg
+            );
+        }
 
-        auto outputHashes = aio.blockOn(staticOutputHashes(*store, drv));
-        std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
-            for (auto & outputName : wantedOutputs) {
-                auto thisOutputHash = outputHashes.at(outputName);
-                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
-                if (!aio.blockOn(store->queryRealisation(thisOutputId))) {
-                    debug("missing output %s", outputName);
-                    assert(optResult);
-                    auto & result = *optResult;
-                    auto i = result.builtOutputs.find(outputName);
-                    assert(i != result.builtOutputs.end());
-                    auto & newRealisation = i->second;
-                    missingRealisations.insert(newRealisation);
-                    missingPaths.insert(newRealisation.outPath);
-                }
-            }
-        } else {
-            auto outputPaths = drv.outputsAndOptPaths(*store);
-            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
-                assert(hopefullyOutputPath.second);
-                if (!aio.blockOn(store->isValidPath(*hopefullyOutputPath.second)))
-                    missingPaths.insert(*hopefullyOutputPath.second);
+        auto outputPaths = drv.outputsAndPaths(*store);
+        for (auto & [outputName, outputPath] : outputPaths) {
+            if (!TRY_AWAIT(store->isValidPath(outputPath.second))) {
+                missingPaths.insert(outputPath.second);
             }
         }
 
@@ -378,19 +531,14 @@ connected:
             if (auto localStore = store.try_cast_shared<LocalStore>())
                 for (auto & path : missingPaths)
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
-            aio.blockOn(
+            TRY_AWAIT(
                 copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute)
             );
         }
-        // XXX: Should be done as part of `copyPaths`
-        for (auto & realisation : missingRealisations) {
-            // Should hold, because if the feature isn't enabled the set
-            // of missing realisations should be empty
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-            aio.blockOn(store->registerDrvOutput(realisation));
-        }
 
-        return 0;
+        context.getResults().initResult().setGood();
+    } catch (...) {
+        RPC_FILL(context.getResults(), initResult, std::current_exception());
     }
 }
 

lix/legacy/nix-build.cc

@@ -272,7 +272,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
             }
             bool add = false;
             if (v.type() == nFunction) {
-                if (auto pattern = dynamic_cast<AttrsPattern *>(v.lambda.fun->pattern.get())) {
+                if (auto pattern = dynamic_cast<AttrsPattern *>(v.lambda().fun->pattern.get())) {
                     for (auto & i : pattern->formals) {
                         if (evaluator->symbols[i.name] == "inNixShell") {
                             add = true;
@@ -285,12 +285,12 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
         };
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(
-                *state,
-                i,
-                takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs,
-                vRoot
-            ).first);
+            Value v(
+                findAlongAttrPath(
+                    *state, i, takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, vRoot
+                )
+                    .first
+            );
             state->forceValue(v, noPos);
             getDerivations(
                 *state,
@@ -355,7 +355,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
                 auto bashDrv = drv->requireDrvPath(*state);
                 pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(bashDrv),
+                    .drvPath = makeConstantStorePath(bashDrv),
                     .outputs = OutputsSpec::Names {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
@@ -368,22 +368,16 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
             }
         }
 
-        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
-
-        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
-            if (!inputNode.value.empty())
+        auto accumDerivedPath = [&](SingleDerivedPath::Opaque inputDrv, const StringSet & inputNode) {
+            if (!inputNode.empty())
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = inputDrv,
-                    .outputs = OutputsSpec::Names { inputNode.value },
+                    .outputs = OutputsSpec::Names { inputNode },
                 });
-            for (const auto & [outputName, childNode] : inputNode.childMap)
-                accumDerivedPath(
-                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
-                    childNode);
         };
 
         // Build or fetch all dependencies of the derivation.
-        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
+        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs) {
             // To get around lambda capturing restrictions in the
             // standard.
             const auto & inputDrv = inputDrv0;
@@ -392,7 +386,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                         return !std::regex_search(store->printStorePath(inputDrv), regex::parse(exclude));
                     }))
             {
-                accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
+                accumDerivedPath(makeConstantStorePath(inputDrv), inputNode);
                 pathsToCopy.insert(inputDrv);
             }
         }
@@ -407,14 +401,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
         if (shellDrv) {
             auto shellDrvOutputs =
-                aio.blockOn(store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore));
-            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
-        }
-
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-            auto resolvedDrv = aio.blockOn(drv.tryResolve(*store));
-            assert(resolvedDrv && "Successfully resolved the derivation");
-            drv = *resolvedDrv;
+                aio.blockOn(store->queryDerivationOutputMap(shellDrv.value(), &*evalStore));
+            shell = store->printStorePath(shellDrvOutputs.at("out")) + "/bin/bash";
         }
 
         // Set the environment.
@@ -455,20 +443,16 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
         if (env.count("__json")) {
             StorePathSet inputs;
 
-            std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
-
-            accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            auto accumInputClosure = [&](const StorePath & inputDrv, const StringSet & inputNode) {
                 auto outputs =
-                    aio.blockOn(store->queryPartialDerivationOutputMap(inputDrv, &*evalStore));
-                for (auto & i : inputNode.value) {
+                    aio.blockOn(store->queryDerivationOutputMap(inputDrv, &*evalStore));
+                for (auto & i : inputNode) {
                     auto o = outputs.at(i);
-                    aio.blockOn(store->computeFSClosure(*o, inputs));
+                    aio.blockOn(store->computeFSClosure(o, inputs));
                 }
-                for (const auto & [outputName, childNode] : inputNode.childMap)
-                    accumInputClosure(*outputs.at(outputName), childNode);
             };
 
-            for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+            for (const auto & [inputDrv, inputNode] : drv.inputDrvs)
                 accumInputClosure(inputDrv, inputNode);
 
             ParsedDerivation parsedDrv(drvInfo.requireDrvPath(*state), drv);
@@ -560,6 +544,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
         logger->pause();
 
+        printMsg(lvlChatty, "running shell: %s", concatMapStringsSep(" ", args, shellEscape));
+
         execvp(shell->c_str(), argPtrs.data());
 
         throw SysError("executing shell '%s'", *shell);
@@ -581,7 +567,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
             pathsToBuild.push_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePathRef(drvPath),
+                .drvPath = makeConstantStorePath(drvPath),
                 .outputs = OutputsSpec::Names{outputName},
             });
             pathsToBuildOrdered.push_back({drvPath, {outputName}});
@@ -607,11 +593,9 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 drvPrefix += fmt("-%d", counter + 1);
 
             auto builtOutputs =
-                aio.blockOn(store->queryPartialDerivationOutputMap(drvPath, &*evalStore));
+                aio.blockOn(store->queryDerivationOutputMap(drvPath, &*evalStore));
 
-            auto maybeOutputPath = builtOutputs.at(outputName);
-            assert(maybeOutputPath);
-            auto outputPath = *maybeOutputPath;
+            auto outputPath = builtOutputs.at(outputName);
 
             if (auto store2 = store.try_cast_shared<LocalFSStore>()) {
                 std::string symlink = drvPrefix;

lix/legacy/nix-channel.cc

@@ -9,6 +9,7 @@
 #include "lix/libstore/temporary-dir.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/regex.hh"
+#include "lix/libutil/result.hh"
 #include "lix/libutil/users.hh"
 #include "nix-channel.hh"
 
@@ -65,13 +66,18 @@ static void addChannel(const std::string & url, const std::string & name)
 static Path profile;
 
 // Remove a channel.
-static void removeChannel(const std::string & name)
-{
+static kj::Promise<Result<void>> removeChannel(const std::string & name)
+try {
     readChannels();
     channels.erase(name);
     writeChannels();
 
-    runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name });
+    TRY_AWAIT(runProgram(
+        settings.nixBinDir + "/nix-env", true, {"--profile", profile, "--uninstall", name}
+    ));
+    co_return result::success();
+} catch (...) {
+    co_return result::current_exception();
 }
 
 static Path nixDefExpr;
@@ -127,8 +133,14 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames)
 
             bool unpacked = false;
             if (std::regex_search(filename, regex::parse("\\.tar\\.(gz|bz2|xz)$"))) {
-                runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
-                            "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
+                aio.blockOn(runProgram(
+                    settings.nixBinDir + "/nix-build",
+                    false,
+                    {"--no-out-link",
+                     "--expr",
+                     "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \""
+                         + name + "\"; src = builtins.storePath \"" + filename + "\"; }"}
+                ));
                 unpacked = true;
             }
 
@@ -158,7 +170,7 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames)
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");
-    runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+    aio.blockOn(runProgram(settings.nixBinDir + "/nix-env", false, envArgs));
 
     // Make the channels appear in nix-env.
     struct stat st;
@@ -244,7 +256,7 @@ static int main_nix_channel(AsyncIoRoot & aio, std::string programName, Strings
             case cRemove:
                 if (args.size() != 1)
                     throw UsageError("'--remove' requires one argument");
-                removeChannel(args[0]);
+                aio.blockOn(removeChannel(args[0]));
                 break;
             case cList:
                 if (!args.empty())
@@ -259,7 +271,11 @@ static int main_nix_channel(AsyncIoRoot & aio, std::string programName, Strings
             case cListGenerations:
                 if (!args.empty())
                     throw UsageError("'--list-generations' expects no arguments");
-                std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush;
+                std::cout << aio.blockOn(runProgram(
+                    settings.nixBinDir + "/nix-env",
+                    false,
+                    {"--profile", profile, "--list-generations"}
+                )) << std::flush;
                 break;
             case cRollback:
                 if (args.size() > 1)
@@ -271,7 +287,7 @@ static int main_nix_channel(AsyncIoRoot & aio, std::string programName, Strings
                 } else {
                     envArgs.push_back("--rollback");
                 }
-                runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+                aio.blockOn(runProgram(settings.nixBinDir + "/nix-env", false, envArgs));
                 break;
         }
 

lix/legacy/nix-collect-garbage.cc

@@ -103,7 +103,7 @@ static int main_nix_collect_garbage(AsyncIoRoot & aio, std::string programName,
         if (dryRun) {
             // Only print results for dry run; when !dryRun, paths will be printed as they're deleted.
             for (auto & i : results.paths) {
-                printInfo("%s", i);
+                printInfo("%s", Uncolored(i));
             }
         }
 

lix/legacy/nix-copy-closure.cc

@@ -23,7 +23,7 @@ static int main_nix_copy_closure(AsyncIoRoot & aio, std::string programName, Str
                 printVersion("nix-copy-closure");
             else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
                 if (*arg != "--gzip")
-                    warn("'%1%' is not implemented, falling back to gzip", *arg);
+                    printTaggedWarning("'%1%' is not implemented, falling back to gzip", *arg);
                 gzip = true;
             } else if (*arg == "--from")
                 toMode = false;

lix/legacy/nix-env.cc

@@ -1,6 +1,7 @@
 #include "lix/libcmd/cmd-profiles.hh"
 #include "lix/libexpr/attr-path.hh"
 #include "lix/libcmd/common-eval-args.hh"
+#include "lix/libexpr/value.hh"
 #include "lix/libstore/derivations.hh"
 #include "lix/libutil/terminal.hh"
 #include "lix/libexpr/eval.hh"
@@ -150,11 +151,12 @@ static void getAllExprs(Evaluator & state,
                 continue;
             }
             /* Load the expression on demand. */
-            auto vArg = state.mem.allocValue();
-            vArg->mkString(path2.canonical().abs());
+            Value vArg;
+            vArg.mkString(path2.canonical().abs());
             if (seen.size() == maxAttrs)
                 throw Error("too many Nix expressions in directory '%1%'", path);
-            attrs.alloc(attrName).mkApp(&state.builtins.get("import"), vArg);
+            attrs.alloc(attrName
+            ) = {NewValueAs::app, state.mem, state.builtins.get("import"), vArg};
         }
         else if (st.type == InputAccessor::tDirectory)
             /* `path2' is a directory (with no default.nix in it);
@@ -181,7 +183,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path_, Value &
        directory). */
     else if (st.type == InputAccessor::tDirectory) {
         auto attrs = state.ctx.buildBindings(maxAttrs);
-        attrs.alloc("_combineChannels").mkList(0);
+        attrs.alloc("_combineChannels") = Value::EMPTY_LIST;
         StringSet seen;
         getAllExprs(state.ctx, path, seen, attrs);
         v.mkAttrs(attrs);
@@ -198,7 +200,7 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath,
     Value vRoot;
     loadSourceExpr(state, nixExprPath, vRoot);
 
-    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first);
+    Value v(findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first);
 
     getDerivations(state, v, pathPrefix, autoArgs, elems, true);
 
@@ -317,9 +319,9 @@ std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches)
     matches.clear();
     for (auto & [name, match] : newest) {
         if (multiple.find(name) != multiple.end())
-            warn(
-                "there are multiple derivations named '%1%'; using the first one",
-                name);
+            printTaggedWarning(
+                "there are multiple derivations named '%1%'; using the first one", name
+            );
         matches.push_back(match);
     }
 
@@ -425,7 +427,7 @@ static void queryInstSources(EvalState & state,
                 Expr & eFun = state.ctx.parseExprFromString(i, CanonPath::fromCwd());
                 Value vFun, vTmp;
                 state.eval(eFun, vFun);
-                vTmp.mkApp(&vFun, &vArg);
+                vTmp = {NewValueAs::app, state.ctx.mem, vFun, vArg};
                 getDerivations(state, vTmp, "", *instSource.autoArgs, elems, true);
             }
 
@@ -480,7 +482,7 @@ static void queryInstSources(EvalState & state,
             Value vRoot;
             loadSourceExpr(state, *instSource.nixExprPath, vRoot);
             for (auto & i : args) {
-                Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first);
+                Value v(findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first);
                 getDerivations(state, v, "", *instSource.autoArgs, elems, true);
             }
             break;
@@ -495,7 +497,7 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath(state))
             targets.emplace_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePathRef(*drvPath),
+                .drvPath = makeConstantStorePath(*drvPath),
                 .outputs = OutputsSpec::All { },
             });
         else
@@ -515,8 +517,8 @@ static bool keep(EvalState & state, DrvInfo & drv)
 static void setMetaFlag(EvalState & state, DrvInfo & drv,
     const std::string & name, const std::string & value)
 {
-    auto v = state.ctx.mem.allocValue();
-    v->mkString(value);
+    Value v;
+    v.mkString(value);
     drv.setMeta(state, name, v);
 }
 
@@ -686,8 +688,12 @@ static void upgradeDerivations(Globals & globals,
                 {
                     const char * action = compareVersions(drvName.version, bestVersion) <= 0
                         ? "upgrading" : "downgrading";
-                    printInfo("%1% '%2%' to '%3%'",
-                        action, i.queryName(*state), bestElem->queryName(*state));
+                    printInfo(
+                        "%1% '%2%' to '%3%'",
+                        Uncolored(action),
+                        i.queryName(*state),
+                        bestElem->queryName(*state)
+                    );
                     newElems.push_back(*bestElem);
                 } else newElems.push_back(i);
 
@@ -792,7 +798,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     std::vector<DerivedPath> paths {
         drvPath
         ? (DerivedPath) (DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(*drvPath),
+            .drvPath = makeConstantStorePath(*drvPath),
             .outputs = OutputsSpec::All { },
         })
         : (DerivedPath) (DerivedPath::Opaque {
@@ -845,7 +851,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
                 );
             }
             if (split == workingElems.end())
-                warn("selector '%s' matched no installed derivations", selector);
+                printTaggedWarning("selector '%s' matched no installed derivations", selector);
             for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) {
                 printInfo("uninstalling '%s'", removedElem->queryName(*state));
             }
@@ -1267,39 +1273,43 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                         else {
                             if (v->type() == nString) {
                                 attrs2["type"] = "string";
-                                attrs2["value"] = v->string.s;
+                                attrs2["value"] = v->str();
                                 xml.writeEmptyElement("meta", attrs2);
                             } else if (v->type() == nInt) {
                                 attrs2["type"] = "int";
-                                attrs2["value"] = fmt("%1%", v->integer);
+                                attrs2["value"] = fmt("%1%", v->integer());
                                 xml.writeEmptyElement("meta", attrs2);
                             } else if (v->type() == nFloat) {
                                 attrs2["type"] = "float";
-                                attrs2["value"] = fmt("%1%", v->fpoint);
+                                attrs2["value"] = fmt("%1%", v->fpoint());
                                 xml.writeEmptyElement("meta", attrs2);
                             } else if (v->type() == nBool) {
                                 attrs2["type"] = "bool";
-                                attrs2["value"] = v->boolean ? "true" : "false";
+                                attrs2["value"] = v->boolean() ? "true" : "false";
                                 xml.writeEmptyElement("meta", attrs2);
                             } else if (v->type() == nList) {
                                 attrs2["type"] = "strings";
                                 XMLOpenElement m(xml, "meta", attrs2);
-                                for (auto elem : v->listItems()) {
-                                    if (elem->type() != nString) continue;
+                                for (auto & elem : v->listItems()) {
+                                    if (elem.type() != nString) {
+                                        continue;
+                                    }
                                     XMLAttrs attrs3;
-                                    attrs3["value"] = elem->string.s;
+                                    attrs3["value"] = elem.str();
                                     xml.writeEmptyElement("string", attrs3);
                                 }
                             } else if (v->type() == nAttrs) {
                                 attrs2["type"] = "strings";
                                 XMLOpenElement m(xml, "meta", attrs2);
-                                Bindings & attrs = *v->attrs;
+                                Bindings & attrs = *v->attrs();
                                 for (auto &i : attrs) {
-                                    Attr & a(*attrs.find(i.name));
-                                    if(a.value->type() != nString) continue;
+                                    const Attr & a(*attrs.get(i.name));
+                                    if (a.value.type() != nString) {
+                                        continue;
+                                    }
                                     XMLAttrs attrs3;
                                     attrs3["type"] = globals.state->symbols[i.name];
-                                    attrs3["value"] = a.value->string.s;
+                                    attrs3["value"] = a.value.str();
                                     xml.writeEmptyElement("string", attrs3);
                             }
                             }

lix/legacy/nix-instantiate.cc

@@ -22,7 +22,7 @@ static Path gcRoot;
 static int rootNr = 0;
 
 
-enum OutputKind { okPlain, okXML, okJSON };
+enum OutputKind { okPlain, okRaw, okXML, okJSON };
 
 void processExpr(EvalState & state, const Strings & attrPaths,
     bool parseOnly, bool strict, Bindings & autoArgs,
@@ -38,7 +38,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
     state.eval(e, vRoot);
 
     for (auto & i : attrPaths) {
-        Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first);
+        Value v(findAlongAttrPath(state, i, autoArgs, vRoot).first);
         state.forceValue(v, noPos);
 
         NixStringContext context;
@@ -48,7 +48,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 vRes = v;
             else
                 state.autoCallFunction(autoArgs, v, vRes, noPos);
-            if (output == okXML)
+            if (output == okRaw)
+                std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output", StringCoercionMode::Strict);
+                // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline
+                // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt.
+            else if (output == okXML)
                 printValueAsXML(state, strict, location, vRes, std::cout, context, noPos);
             else if (output == okJSON) {
                 printValueAsJSON(state, strict, vRes, noPos, std::cout, context);
@@ -130,6 +134,8 @@ static int main_nix_instantiate(AsyncIoRoot & aio, std::string programName, Stri
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
                 ;
+            else if (*arg == "--raw")
+                outputKind = okRaw;
             else if (*arg == "--xml")
                 outputKind = okXML;
             else if (*arg == "--json")

lix/legacy/nix-store.cc

@@ -17,8 +17,11 @@
 #include "graphml.hh"
 #include "lix/libcmd/legacy.hh"
 #include "lix/libstore/path-with-outputs.hh"
+#include "lix/libutil/serialise.hh"
 #include "nix-store.hh"
 
+#include <cstdint>
+#include <ctime>
 #include <iostream>
 #include <algorithm>
 
@@ -33,25 +36,23 @@ namespace nix {
 using std::cin;
 using std::cout;
 
-
-typedef void (* Operation) (AsyncIoRoot & aio, Strings opFlags, Strings opArgs);
-
+typedef void (*Operation)(
+    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
+);
 
 static Path gcRoot;
 static int rootNr = 0;
 static bool noOutput = false;
-static std::shared_ptr<Store> store;
 
-
-ref<LocalStore> ensureLocalStore()
+ref<LocalStore> ensureLocalStore(std::shared_ptr<Store> store)
 {
     auto store2 = std::dynamic_pointer_cast<LocalStore>(store);
     if (!store2) throw Error("you don't have sufficient rights to use this command");
     return ref<LocalStore>::unsafeFromPtr(store2);
 }
 
-
-static kj::Promise<Result<StorePath>> useDeriver(const StorePath & path)
+static kj::Promise<Result<StorePath>>
+useDeriver(std::shared_ptr<Store> store, const StorePath & path)
 try {
     if (path.isDerivation()) co_return path;
     auto info = TRY_AWAIT(store->queryPathInfo(path));
@@ -65,7 +66,8 @@ try {
 
 /* Realise the given path.  For a derivation that means build it; for
    other paths it means ensure their validity. */
-static kj::Promise<Result<PathSet>> realisePath(StorePathWithOutputs path, bool build = true)
+static kj::Promise<Result<PathSet>>
+realisePath(std::shared_ptr<Store> store, StorePathWithOutputs path, bool build = true)
 try {
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
@@ -125,7 +127,8 @@ try {
 
 
 /* Realise the given paths. */
-static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRealise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool dryRun = false;
     BuildMode buildMode = bmNormal;
@@ -170,7 +173,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
-            auto paths2 = aio.blockOn(realisePath(i, false));
+            auto paths2 = aio.blockOn(realisePath(store, i, false));
             if (!noOutput)
                 for (auto & j : paths2)
                     cout << fmt("%1%\n", j);
@@ -179,7 +182,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Add files to the Nix store and print the resulting paths. */
-static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opAdd(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -196,7 +199,8 @@ static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
-static void opAddFixed(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opAddFixed(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -222,7 +226,8 @@ static void opAddFixed(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Hack to support caching in `nix-prefetch-url'. */
-static void opPrintFixedPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opPrintFixedPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -245,19 +250,20 @@ static void opPrintFixedPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     })));
 }
 
-
-static kj::Promise<Result<StorePathSet>> maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise)
+static kj::Promise<Result<StorePathSet>> maybeUseOutputs(
+    std::shared_ptr<Store> store, const StorePath & storePath, bool useOutput, bool forceRealise
+)
 try {
-    if (forceRealise) TRY_AWAIT(realisePath({storePath}));
+    if (forceRealise) {
+        TRY_AWAIT(realisePath(store, {storePath}));
+    }
     if (useOutput && storePath.isDerivation()) {
         auto drv = TRY_AWAIT(store->derivationFromPath(storePath));
         StorePathSet outputs;
         if (forceRealise)
             co_return TRY_AWAIT(store->queryDerivationOutputs(storePath));
-        for (auto & i : drv.outputsAndOptPaths(*store)) {
-            if (!i.second.second)
-                throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
-            outputs.insert(*i.second.second);
+        for (auto & i : drv.outputsAndPaths(*store)) {
+            outputs.insert(i.second.second);
         }
         co_return outputs;
     }
@@ -270,8 +276,14 @@ try {
 /* Some code to print a tree representation of a derivation dependency
    graph.  Topological sorting is used to keep the tree relatively
    flat. */
-static void printTree(AsyncIoRoot & aio, const StorePath & path,
-    const std::string & firstPad, const std::string & tailPad, StorePathSet & done)
+static void printTree(
+    std::shared_ptr<Store> store,
+    AsyncIoRoot & aio,
+    const StorePath & path,
+    const std::string & firstPad,
+    const std::string & tailPad,
+    StorePathSet & done
+)
 {
     if (!done.insert(path).second) {
         cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path));
@@ -291,16 +303,21 @@ static void printTree(AsyncIoRoot & aio, const StorePath & path,
 
     for (const auto &[n, i] : enumerate(sorted)) {
         bool last = n + 1 == sorted.size();
-        printTree(aio, i,
+        printTree(
+            store,
+            aio,
+            i,
             tailPad + (last ? treeLast : treeConn),
             tailPad + (last ? treeNull : treeLine),
-            done);
+            done
+        );
     }
 }
 
 
 /* Perform various sorts of queries. */
-static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     enum QueryType
         { qOutputs, qRequisites, qReferences, qReferrers
@@ -351,7 +368,9 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                auto outputs = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise));
+                auto outputs = aio.blockOn(
+                    maybeUseOutputs(store, store->followLinksToStorePath(i), true, forceRealise)
+                );
                 for (auto & outputPath : outputs)
                     cout << fmt("%1%\n", store->printStorePath(outputPath));
             }
@@ -364,7 +383,9 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qReferrersClosure: {
             StorePathSet paths;
             for (auto & i : opArgs) {
-                auto ps = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise));
+                auto ps = aio.blockOn(maybeUseOutputs(
+                    store, store->followLinksToStorePath(i), useOutput, forceRealise
+                ));
                 for (auto & j : ps) {
                     if (query == qRequisites) {
                         aio.blockOn(store->computeFSClosure(j, paths, false, includeOutputs));
@@ -415,7 +436,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
         case qBinding:
             for (auto & i : opArgs) {
-                auto path = aio.blockOn(useDeriver(store->followLinksToStorePath(i)));
+                auto path = aio.blockOn(useDeriver(store, store->followLinksToStorePath(i)));
                 Derivation drv = aio.blockOn(store->derivationFromPath(path));
                 StringPairs::iterator j = drv.env.find(bindingName);
                 if (j == drv.env.end())
@@ -428,7 +449,10 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qHash:
         case qSize:
             for (auto & i : opArgs) {
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))) {
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     auto info = aio.blockOn(store->queryPathInfo(j));
                     if (query == qHash) {
                         assert(info->narHash.type == HashType::SHA256);
@@ -442,15 +466,19 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qTree: {
             StorePathSet done;
             for (auto & i : opArgs)
-                printTree(aio, store->followLinksToStorePath(i), "", "", done);
+                printTree(store, aio, store->followLinksToStorePath(i), "", "", done);
             break;
         }
 
         case qGraph: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     roots.insert(j);
+                }
             aio.blockOn(printDotGraph(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -458,8 +486,12 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qGraphML: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     roots.insert(j);
+                }
             aio.blockOn(printGraphML(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -473,8 +505,12 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qRoots: {
             StorePathSet args;
             for (auto & i : opArgs)
-                for (auto & p : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & p : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     args.insert(p);
+                }
 
             StorePathSet referrers;
             aio.blockOn(store->computeFSClosure(
@@ -494,8 +530,8 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void opPrintEnv(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opPrintEnv(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path");
@@ -520,8 +556,8 @@ static void opPrintEnv(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     cout << "'\n";
 }
 
-
-static void opReadLog(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opReadLog(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -538,8 +574,8 @@ static void opReadLog(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void opDumpDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opDumpDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty()) {
@@ -554,8 +590,13 @@ static void opDumpDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void registerValidity(AsyncIoRoot & aio, bool reregister, bool hashGiven, bool canonicalise)
+static void registerValidity(
+    std::shared_ptr<Store> store,
+    AsyncIoRoot & aio,
+    bool reregister,
+    bool hashGiven,
+    bool canonicalise
+)
 {
     ValidPathInfos infos;
 
@@ -578,20 +619,20 @@ static void registerValidity(AsyncIoRoot & aio, bool reregister, bool hashGiven,
         }
     }
 
-    aio.blockOn(ensureLocalStore()->registerValidPaths(infos));
+    aio.blockOn(ensureLocalStore(store)->registerValidPaths(infos));
 }
 
-
-static void opLoadDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opLoadDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
-    registerValidity(aio, true, true, false);
+    registerValidity(store, aio, true, true, false);
 }
 
-
-static void opRegisterValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRegisterValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool reregister = false; // !!! maybe this should be the default
     bool hashGiven = false;
@@ -603,11 +644,11 @@ static void opRegisterValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArg
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
-    registerValidity(aio, reregister, hashGiven, true);
+    registerValidity(store, aio, reregister, hashGiven, true);
 }
 
-
-static void opCheckValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opCheckValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printInvalid = false;
 
@@ -626,8 +667,7 @@ static void opCheckValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void opGC(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opGC(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printRoots = false;
     GCOptions options;
@@ -673,7 +713,8 @@ static void opGC(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 /* Remove paths from the Nix store if possible (i.e., if they do not
    have any remaining referrers and are not reachable from any GC
    roots). */
-static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opDelete(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     GCOptions options;
     options.action = GCOptions::gcDeleteSpecific;
@@ -703,7 +744,7 @@ static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Dump a path as a Nix archive.  The archive is written to stdout */
-static void opDump(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opDump(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -716,7 +757,8 @@ static void opDump(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Restore a value from a Nix archive.  The archive is read from stdin. */
-static void opRestore(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRestore(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -725,8 +767,8 @@ static void opRestore(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     restorePath(*opArgs.begin(), source);
 }
 
-
-static void opExport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opExport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -741,8 +783,8 @@ static void opExport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     sink.flush();
 }
 
-
-static void opImport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opImport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -758,7 +800,7 @@ static void opImport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Initialise the Nix databases. */
-static void opInit(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opInit(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
@@ -769,7 +811,8 @@ static void opInit(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Verify the consistency of the Nix environment. */
-static void opVerify(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opVerify(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
@@ -783,14 +826,15 @@ static void opVerify(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", i);
 
     if (aio.blockOn(store->verifyStore(checkContents, repair))) {
-        warn("not all store errors were fixed");
+        printTaggedWarning("not all store errors were fixed");
         throw Exit(1);
     }
 }
 
 
 /* Verify whether the contents of the given store path have not changed. */
-static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opVerifyPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -802,7 +846,7 @@ static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = aio.blockOn(store->queryPathInfo(path));
         HashSink sink(info->narHash.type);
-        aio.blockOn(store->narFromPath(path))->drainInto(sink);
+        aio.blockOn(aio.blockOn(store->narFromPath(path))->drainInto(sink));
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
@@ -819,7 +863,8 @@ static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Repair the contents of the given path by redownloading it using a
    substituter (if available). */
-static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRepairPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -830,7 +875,8 @@ static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Optimise the disk space usage of the Nix store by hard-linking
    files with the same contents. */
-static void opOptimise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opOptimise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty() || !opFlags.empty())
         throw UsageError("no arguments expected");
@@ -839,7 +885,8 @@ static void opOptimise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 }
 
 /* Serve the nix store in a way usable by a restricted ssh user. */
-static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool writeAllowed = false;
     for (auto & i : opFlags)
@@ -852,17 +899,19 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     FdSink out(STDOUT_FILENO);
 
     /* Exchange the greeting. */
-    unsigned int magic = readInt(in);
+    unsigned int magic = readNum<unsigned>(in);
     if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
     out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION;
     out.flush();
-    ServeProto::Version clientVersion = readInt(in);
+    ServeProto::Version clientVersion = readNum<unsigned>(in);
 
     ServeProto::ReadConn rconn {
         .from = in,
+        .store = *store,
         .version = clientVersion,
     };
     ServeProto::WriteConn wconn {
+        .store = *store,
         .version = clientVersion,
     };
 
@@ -872,12 +921,12 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         verbosity = lvlError;
         settings.keepLog.override(false);
         settings.useSubstitutes.override(false);
-        settings.maxSilentTime.override(readInt(in));
-        settings.buildTimeout.override(readInt(in));
+        settings.maxSilentTime.override(readNum<unsigned>(in));
+        settings.buildTimeout.override(readNum<unsigned>(in));
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
             settings.maxLogSize.override(readNum<unsigned long>(in));
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
-            auto nrRepeats = readInt(in);
+            auto nrRepeats = readNum<unsigned>(in);
             if (nrRepeats != 0) {
                 throw Error("client requested repeating builds, but this is not currently implemented");
             }
@@ -887,19 +936,19 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             // `nrRepeats` in fact is 0, so we can safely ignore this
             // without doing something other than what the client
             // asked for.
-            readInt(in);
+            readNum<unsigned>(in);
 
             settings.runDiffHook.override(true);
         }
         if (GET_PROTOCOL_MINOR(clientVersion) >= 7) {
-            settings.keepFailed.override((bool) readInt(in));
+            settings.keepFailed.override((bool) readNum<unsigned>(in));
         }
     };
 
     while (true) {
         ServeProto::Command cmd;
         try {
-            cmd = (ServeProto::Command) readInt(in);
+            cmd = (ServeProto::Command) readNum<unsigned>(in);
         } catch (EndOfFile & e) {
             break;
         }
@@ -907,9 +956,9 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         switch (cmd) {
 
             case ServeProto::Command::QueryValidPaths: {
-                bool lock = readInt(in);
-                bool substitute = readInt(in);
-                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
+                bool lock = readNum<unsigned>(in);
+                bool substitute = readNum<unsigned>(in);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         aio.blockOn(store->addTempRoot(path));
@@ -919,18 +968,18 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 }
 
                 auto valid = aio.blockOn(store->queryValidPaths(paths));
-                out << ServeProto::write(*store, wconn, valid);
+                out << ServeProto::write(wconn, valid);
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = aio.blockOn(store->queryPathInfo(i));
                         out << store->printStorePath(info->path);
-                        out << ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
+                        out << ServeProto::write(wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
                     } catch (InvalidPath &) {
                     }
                 }
@@ -939,8 +988,8 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             }
 
             case ServeProto::Command::DumpStorePath:
-                aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
-                    ->drainInto(out);
+                aio.blockOn(aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
+                                ->drainInto(out));
                 break;
 
             case ServeProto::Command::ImportPaths: {
@@ -951,9 +1000,9 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             }
 
             case ServeProto::Command::ExportPaths: {
-                readInt(in); // obsolete
+                readNum<unsigned>(in); // obsolete
                 aio.blockOn(store->exportPaths(
-                    ServeProto::Serialise<StorePathSet>::read(*store, rconn), out
+                    ServeProto::Serialise<StorePathSet>::read(rconn), out
                 ));
                 break;
             }
@@ -992,20 +1041,20 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 MonitorFdHup monitor(in.fd);
                 auto status = aio.blockOn(store->buildDerivation(drvPath, drv));
 
-                out << ServeProto::write(*store, wconn, status);
+                out << ServeProto::write(wconn, status);
                 break;
             }
 
             case ServeProto::Command::QueryClosure: {
-                bool includeOutputs = readInt(in);
+                bool includeOutputs = readNum<unsigned>(in);
                 StorePathSet closure;
                 aio.blockOn(store->computeFSClosure(
-                    ServeProto::Serialise<StorePathSet>::read(*store, rconn),
+                    ServeProto::Serialise<StorePathSet>::read(rconn),
                     closure,
                     false,
                     includeOutputs
                 ));
-                out << ServeProto::write(*store, wconn, closure);
+                out << ServeProto::write(wconn, closure);
                 break;
             }
 
@@ -1020,14 +1069,44 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
-                in >> info.registrationTime >> info.narSize >> info.ultimate;
+                info.references = ServeProto::Serialise<StorePathSet>::read(rconn);
+                info.registrationTime = readNum<time_t>(in);
+                info.narSize = readNum<uint64_t>(in);
+                info.ultimate = readBool(in);
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));
 
                 if (info.narSize == 0)
                     throw Error("narInfo is too old and missing the narSize field");
 
+                struct SizedSource : Source
+                {
+                    Source & orig;
+                    size_t remain;
+                    SizedSource(Source & orig, size_t size) : orig(orig), remain(size) {}
+                    size_t read(char * data, size_t len) override
+                    {
+                        if (this->remain <= 0) {
+                            throw EndOfFile("sized: unexpected end-of-file");
+                        }
+                        len = std::min(len, this->remain);
+                        size_t n = this->orig.read(data, len);
+                        this->remain -= n;
+                        return n;
+                    }
+
+                    size_t drainAll()
+                    {
+                        std::vector<char> buf(8192);
+                        size_t sum = 0;
+                        while (this->remain > 0) {
+                            size_t n = read(buf.data(), buf.size());
+                            sum += n;
+                        }
+                        return sum;
+                    }
+                };
+
                 SizedSource sizedSource(in, info.narSize);
                 AsyncSourceInputStream stream{sizedSource};
 
@@ -1049,8 +1128,9 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void opGenerateBinaryCacheKey(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opGenerateBinaryCacheKey(
+    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
+)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -1068,8 +1148,8 @@ static void opGenerateBinaryCacheKey(AsyncIoRoot & aio, Strings opFlags, Strings
     writeFile(secretKeyFile, secretKey.to_string());
 }
 
-
-static void opVersion(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opVersion(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     printVersion("nix-store");
 }
@@ -1214,10 +1294,11 @@ static int main_nix_store(AsyncIoRoot & aio, std::string programName, Strings ar
         if (showHelp) showManPage("nix-store" + opName);
         if (!op) throw UsageError("no operation specified");
 
+        std::shared_ptr<Store> store;
         if (op != opDump && op != opRestore) /* !!! hack */
             store = aio.blockOn(openStore());
 
-        op(aio, std::move(opFlags), std::move(opArgs));
+        op(store, aio, std::move(opFlags), std::move(opArgs));
 
         return 0;
     }

lix/legacy/user-env.cc

@@ -1,4 +1,5 @@
 #include "user-env.hh"
+#include "lix/libexpr/value.hh"
 #include "lix/libstore/derivations.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libstore/path-with-outputs.hh"
@@ -32,7 +33,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
     /* Construct the whole top level derivation. */
     StorePathSet references;
-    Value manifest = state.ctx.mem.newList(elems.size());
+    auto manifest = state.ctx.mem.newList(elems.size());
+    Value vManifest{NewValueAs::list, manifest};
     size_t n = 0;
     for (auto & i : elems) {
         /* Create a pseudo-derivation containing the name, system,
@@ -55,9 +57,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         // Copy each output meant for installation.
         auto & vOutputs = attrs.alloc(state.ctx.s.outputs);
-        vOutputs = state.ctx.mem.newList(outputs.size());
+        auto outputsList = state.ctx.mem.newList(outputs.size());
+        vOutputs = {NewValueAs::list, outputsList};
         for (const auto & [m, j] : enumerate(outputs)) {
-            (vOutputs.listElems()[m] = state.ctx.mem.allocValue())->mkString(j.first);
+            outputsList->elems[m].mkString(j.first);
             auto outputAttrs = state.ctx.buildBindings(2);
             outputAttrs.alloc(state.ctx.s.outPath).mkString(state.ctx.store->printStorePath(*j.second));
             attrs.alloc(j.first).mkAttrs(outputAttrs);
@@ -75,12 +78,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         for (auto & j : metaNames) {
             Value * v = i.queryMeta(state, j);
             if (!v) continue;
-            meta.insert(state.ctx.symbols.create(j), v);
+            meta.insert(state.ctx.symbols.create(j), *v);
         }
 
         attrs.alloc(state.ctx.s.meta).mkAttrs(meta);
 
-        (manifest.listElems()[n++] = state.ctx.mem.allocValue())->mkAttrs(attrs);
+        manifest->elems[n++].mkAttrs(attrs);
 
         if (drvPath) references.insert(*drvPath);
     }
@@ -89,7 +92,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
        the store; we need it for future modifications of the
        environment. */
     std::ostringstream str;
-    printAmbiguous(manifest, state.ctx.symbols, str, nullptr, std::numeric_limits<int>::max());
+    printAmbiguous(vManifest, state.ctx.symbols, str, nullptr, std::numeric_limits<int>::max());
     auto manifestFile = state.aio.blockOn(state.ctx.store->addTextToStore("env-manifest.nix",
         str.str(), references));
 
@@ -103,21 +106,20 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
        builder with the manifest as argument. */
     auto attrs = state.ctx.buildBindings(3);
     state.ctx.paths.mkStorePathString(manifestFile, attrs.alloc("manifest"));
-    attrs.insert(state.ctx.symbols.create("derivations"), &manifest);
+    attrs.insert(state.ctx.symbols.create("derivations"), vManifest);
     Value args;
     args.mkAttrs(attrs);
 
-    Value topLevel;
-    topLevel.mkApp(&envBuilder, &args);
+    Value topLevel{NewValueAs::app, state.ctx.mem, envBuilder, args};
 
     /* Evaluate it. */
     debug("evaluating user environment builder");
     state.forceValue(topLevel, noPos);
     NixStringContext context;
-    Attr & aDrvPath(*topLevel.attrs->find(state.ctx.s.drvPath));
-    auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, "");
-    Attr & aOutPath(*topLevel.attrs->find(state.ctx.s.outPath));
-    auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, "");
+    const Attr & aDrvPath(*topLevel.attrs()->get(state.ctx.s.drvPath));
+    auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, aDrvPath.value, context, "");
+    const Attr & aOutPath(*topLevel.attrs()->get(state.ctx.s.outPath));
+    auto topLevelOut = state.coerceToStorePath(aOutPath.pos, aOutPath.value, context, "");
 
     /* Realise the resulting store expression. */
     debug("building user environment");

lix/libcmd/built-path.cc

@@ -13,9 +13,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -23,10 +23,6 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
-
 #define FIELD_TYPE std::map<std::string, StorePath>
 CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef FIELD_TYPE
@@ -34,16 +30,6 @@ CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
-StorePath SingleBuiltPath::outPath() const
-{
-    return std::visit(
-        overloaded{
-            [](const SingleBuiltPath::Opaque & p) { return p.path; },
-            [](const SingleBuiltPath::Built & b) { return b.output.second; },
-        }, raw()
-    );
-}
-
 StorePathSet BuiltPath::outPaths() const
 {
     return std::visit(
@@ -59,32 +45,10 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
-SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
-{
-    return SingleDerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
-        .output = output.first,
-    };
-}
-
-SingleDerivedPath SingleBuiltPath::discardOutputPath() const
-{
-    return std::visit(
-        overloaded{
-            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
-                return p;
-            },
-            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
-                return b.discardOutputPath();
-            },
-        }, raw()
-    );
-}
-
 kj::Promise<Result<JSON>> BuiltPath::Built::toJSON(const Store & store) const
 try {
     JSON res;
-    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
+    res["drvPath"] = TRY_AWAIT(drvPath.toJSON(store));
     for (const auto & [outputName, outputPath] : outputs) {
         res["outputs"][outputName] = store.printStorePath(outputPath);
     }
@@ -93,36 +57,6 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<JSON>> SingleBuiltPath::Built::toJSON(const Store & store) const
-try {
-    JSON res;
-    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
-    auto & [outputName, outputPath] = output;
-    res["output"] = outputName;
-    res["outputPath"] = store.printStorePath(outputPath);
-    co_return res;
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<JSON>> SingleBuiltPath::toJSON(const Store & store) const
-try {
-    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
-        return buildable.toJSON(store);
-    }, raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
-
-
-kj::Promise<Result<JSON>> BuiltPath::toJSON(const Store & store) const
-try {
-    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
-        return buildable.toJSON(store);
-    }, raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
 
 kj::Promise<Result<RealisedPath::Set>> BuiltPath::toRealisedPaths(Store & store) const
 try {
@@ -141,24 +75,10 @@ try {
         [&](const BuiltPath::Built & p) -> kj::Promise<Result<void>> {
             try {
                 auto drvHashes = TRY_AWAIT(
-                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath->outPath())))
+                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath.path)))
                 );
                 for (auto& [outputName, outputPath] : p.outputs) {
-                    if (experimentalFeatureSettings.isEnabled(
-                                Xp::CaDerivations)) {
-                        auto drvOutput = get(drvHashes, outputName);
-                        if (!drvOutput)
-                            throw Error(
-                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
-                                store.printStorePath(p.drvPath->outPath()), outputName);
-                        auto thisRealisation = TRY_AWAIT(store.queryRealisation(
-                            DrvOutput{*drvOutput, outputName}));
-                        assert(thisRealisation);  // We’ve built it, so we must
-                                                  // have the realisation
-                        res.insert(*thisRealisation);
-                    } else {
-                        res.insert(outputPath);
-                    }
+                    res.insert(outputPath);
                 }
                 co_return result::success();
             } catch (...) {

lix/libcmd/built-path.hh

@@ -7,63 +7,15 @@
 
 namespace nix {
 
-struct SingleBuiltPath;
-
-struct SingleBuiltPathBuilt {
-    ref<SingleBuiltPath> drvPath;
-    std::pair<std::string, StorePath> output;
-
-    SingleDerivedPathBuilt discardOutputPath() const;
-
-    std::string to_string(const Store & store) const;
-    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
-
-    DECLARE_CMP(SingleBuiltPathBuilt);
-};
-
-namespace built_path::detail {
-using SingleBuiltPathRaw = std::variant<
-    DerivedPathOpaque,
-    SingleBuiltPathBuilt
->;
-}
-
-struct SingleBuiltPath : built_path::detail::SingleBuiltPathRaw {
-    using Raw = built_path::detail::SingleBuiltPathRaw;
-    using Raw::Raw;
-
-    using Opaque = DerivedPathOpaque;
-    using Built = SingleBuiltPathBuilt;
-
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
-
-    StorePath outPath() const;
-
-    SingleDerivedPath discardOutputPath() const;
-
-    static SingleBuiltPath parse(const Store & store, std::string_view);
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
-};
-
-static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
-{
-    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
-}
-
 /**
  * A built derived path with hints in the form of optional concrete output paths.
  *
  * See 'BuiltPath' for more an explanation.
  */
 struct BuiltPathBuilt {
-    ref<SingleBuiltPath> drvPath;
+    DerivedPathOpaque drvPath;
     std::map<std::string, StorePath> outputs;
 
-    std::string to_string(const Store & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
@@ -93,8 +45,6 @@ struct BuiltPath : built_path::detail::BuiltPathRaw {
 
     StorePathSet outPaths() const;
     kj::Promise<Result<RealisedPath::Set>> toRealisedPaths(Store & store) const;
-
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

lix/libcmd/command.cc

@@ -4,6 +4,7 @@
 #include "lix/libstore/derivations.hh"
 #include "lix/libstore/profiles.hh"
 #include "lix/libcmd/repl.hh"
+#include "lix/libutil/async.hh"
 
 extern char * * environ __attribute__((weak));
 
@@ -39,14 +40,15 @@ StoreCommand::StoreCommand()
 
 ref<Store> StoreCommand::getStore()
 {
-    if (!_store)
-        _store = createStore();
+    if (!_store) {
+        _store = createStore(aio());
+    }
     return *_store;
 }
 
-ref<Store> StoreCommand::createStore()
+ref<Store> StoreCommand::createStore(AsyncIoRoot & in)
 {
-    return aio().blockOn(openStore());
+    return in.blockOn(openStore());
 }
 
 void StoreCommand::run()
@@ -71,9 +73,9 @@ CopyCommand::CopyCommand()
     });
 }
 
-ref<Store> CopyCommand::createStore()
+ref<Store> CopyCommand::createStore(AsyncIoRoot & in)
 {
-    return srcUri.empty() ? StoreCommand::createStore() : aio().blockOn(openStore(srcUri));
+    return srcUri.empty() ? StoreCommand::createStore(in) : in.blockOn(openStore(srcUri));
 }
 
 ref<Store> CopyCommand::getDstStore()

lix/libcmd/command.hh

@@ -39,7 +39,7 @@ struct StoreCommand : virtual Command
     StoreCommand();
     void run() override;
     ref<Store> getStore();
-    virtual ref<Store> createStore();
+    virtual ref<Store> createStore(AsyncIoRoot & in);
     /**
      * Main entry point, with a `Store` provided
      */
@@ -59,7 +59,7 @@ struct CopyCommand : virtual StoreCommand
 
     CopyCommand();
 
-    ref<Store> createStore() override;
+    ref<Store> createStore(AsyncIoRoot & in) override;
 
     ref<Store> getDstStore();
 };

lix/libcmd/common-eval-args.cc

@@ -9,6 +9,7 @@
 #include "lix/libstore/store-api.hh"
 #include "lix/libcmd/command.hh"
 #include "lix/libutil/async.hh"
+#include "lix/libutil/error.hh"
 #include "lix/libutil/regex.hh"
 
 #include <regex>
@@ -16,31 +17,36 @@
 namespace nix {
 
 static std::regex const identifierRegex = regex::parse("^[A-Za-z_][A-Za-z0-9_'-]*$");
-static void warnInvalidNixIdentifier(const std::string & name)
+static void checkValidNixIdentifier(const std::string & name)
 {
     std::smatch match;
     if (!std::regex_match(name, match, identifierRegex)) {
-        warn("This Nix invocation specifies a value for argument '%s' which isn't a valid \
-Nix identifier. The project is considering to drop support for this \
-or to require quotes around args that aren't valid Nix identifiers. \
-If you depend on this behvior, please reach out in \
-https://git.lix.systems/lix-project/lix/issues/496 so we can discuss \
-your use-case.", name);
+        throw UsageError(
+            "This invocation specifies a value for argument '%s' "
+            "which isn't a valid Nix identifier. "
+            "The project is dropping support for this so that it's possible to make e.g. "
+            "'%s' evaluating to '%s' in the future. "
+            "If you depend on this behavior, please reach out in "
+            "<https://git.lix.systems/lix-project/lix/issues/496> so we can discuss your use-case.",
+            name,
+            "--arg config.allowUnfree true",
+            "{ config.allowUnfree = true; }"
+        );
     }
 }
 
 MixEvalArgs::MixEvalArgs()
 {
-    addFlag({
-        .longName = "arg",
-        .description = "Pass the value *expr* as the argument *name* to Nix functions.",
-        .category = category,
-        .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) {
-            warnInvalidNixIdentifier(name);
-            autoArgs[name] = 'E' + expr;
-        }}
-    });
+    addFlag(
+        {.longName = "arg",
+         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
+         .category = category,
+         .labels = {"name", "expr"},
+         .handler = {[&](std::string name, std::string expr) {
+             checkValidNixIdentifier(name);
+             autoArgs[name] = 'E' + expr;
+         }}}
+    );
 
     addFlag({
         .longName = "argstr",
@@ -48,7 +54,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) {
-            warnInvalidNixIdentifier(name);
+            checkValidNixIdentifier(name);
             autoArgs[name] = 'S' + s;
         }},
     });
@@ -177,13 +183,13 @@ Bindings * MixEvalArgs::getAutoArgs(Evaluator & state)
 {
     auto res = state.buildBindings(autoArgs.size());
     for (auto & i : autoArgs) {
-        auto v = state.mem.allocValue();
+        Value v;
         if (i.second[0] == 'E')
             state.evalLazily(
-                state.parseExprFromString(i.second.substr(1), CanonPath::fromCwd()), *v
+                state.parseExprFromString(i.second.substr(1), CanonPath::fromCwd()), v
             );
         else
-            v->mkString(((std::string_view) i.second).substr(1));
+            v.mkString(((std::string_view) i.second).substr(1));
         res.insert(state.symbols.create(i.first), v);
     }
     return res.finish();

lix/libcmd/installable-attr-path.cc

@@ -12,9 +12,10 @@ namespace nix {
 InstallableAttrPath::InstallableAttrPath(
     ref<eval_cache::CachingEvaluator> state,
     SourceExprCommand & cmd,
-    Value * v,
+    Value & v,
     const std::string & attrPath,
-    ExtendedOutputsSpec extendedOutputsSpec)
+    ExtendedOutputsSpec extendedOutputsSpec
+)
     : InstallableValue(state)
     , cmd(cmd)
     , v(allocRootValue(v))
@@ -22,10 +23,10 @@ InstallableAttrPath::InstallableAttrPath(
     , extendedOutputsSpec(std::move(extendedOutputsSpec))
 { }
 
-std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
+std::pair<Value, PosIdx> InstallableAttrPath::toValue(EvalState & state)
 {
-    auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(*evaluator), **v);
-    state.forceValue(*vRes, pos);
+    auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(*evaluator), *v);
+    state.forceValue(vRes, pos);
     return {vRes, pos};
 }
 
@@ -34,7 +35,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
     auto [v, pos] = toValue(state);
 
     if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
-            state, *v, pos, fmt("while evaluating the attribute '%s'", attrPath)
+            state, v, pos, fmt("while evaluating the attribute '%s'", attrPath)
         ))
     {
         return { *derivedPathWithInfo };
@@ -43,7 +44,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
     Bindings & autoArgs = *cmd.getAutoArgs(*evaluator);
 
     DrvInfos drvInfos;
-    getDerivations(state, *v, "", autoArgs, drvInfos, false);
+    getDerivations(state, v, "", autoArgs, drvInfos, false);
 
     // Backward compatibility hack: group results by drvPath. This
     // helps keep .all output together.
@@ -76,7 +77,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = makeConstantStorePathRef(drvPath),
+                .drvPath = makeConstantStorePath(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {
@@ -92,9 +93,10 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
 InstallableAttrPath InstallableAttrPath::parse(
     ref<eval_cache::CachingEvaluator> state,
     SourceExprCommand & cmd,
-    Value * v,
+    Value & v,
     std::string_view prefix,
-    ExtendedOutputsSpec extendedOutputsSpec)
+    ExtendedOutputsSpec extendedOutputsSpec
+)
 {
     return {
         state, cmd, v,

lix/libcmd/installable-attr-path.hh

@@ -20,13 +20,14 @@ class InstallableAttrPath : public InstallableValue
     InstallableAttrPath(
         ref<eval_cache::CachingEvaluator> state,
         SourceExprCommand & cmd,
-        Value * v,
+        Value & v,
         const std::string & attrPath,
-        ExtendedOutputsSpec extendedOutputsSpec);
+        ExtendedOutputsSpec extendedOutputsSpec
+    );
 
     std::string what() const override { return attrPath; };
 
-    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+    std::pair<Value, PosIdx> toValue(EvalState & state) override;
 
     DerivedPathsWithInfo toDerivedPaths(EvalState & state) override;
 
@@ -35,9 +36,10 @@ public:
     static InstallableAttrPath parse(
         ref<eval_cache::CachingEvaluator> state,
         SourceExprCommand & cmd,
-        Value * v,
+        Value & v,
         std::string_view prefix,
-        ExtendedOutputsSpec extendedOutputsSpec);
+        ExtendedOutputsSpec extendedOutputsSpec
+    );
 };
 
 }

lix/libcmd/installable-derived-path.cc

@@ -26,36 +26,27 @@ InstallableDerivedPath InstallableDerivedPath::parse(
     std::string_view prefix,
     ExtendedOutputsSpec extendedOutputsSpec)
 {
-    auto derivedPath = std::visit(overloaded {
-        // If the user did not use ^, we treat the output more
-        // liberally: we accept a symlink chain or an actual
-        // store path.
-        [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
-            auto storePath = store->followLinksToStorePath(prefix);
-            // Remove this prior to stabilizing the new CLI.
-            if (storePath.isDerivation()) {
-                auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(storePath),
-                    .outputs = OutputsSpec::All { },
+    auto derivedPath = std::visit(
+        overloaded{
+            // If the user did not use ^, we treat the output more
+            // liberally: we accept a symlink chain or an actual
+            // store path.
+            [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
+                return DerivedPath::Opaque{
+                    .path = store->followLinksToStorePath(prefix),
                 };
-                warn(
-                    "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'",
-                    oldDerivedPath.to_string(*store));
-            };
-            return DerivedPath::Opaque {
-                .path = std::move(storePath),
-            };
+            },
+            // If the user did use ^, we just do exactly what is written.
+            [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+                auto drv = DerivedPathOpaque::parse(*store, prefix);
+                return DerivedPath::Built{
+                    .drvPath = std::move(drv),
+                    .outputs = outputSpec,
+                };
+            },
         },
-        // If the user did use ^, we just do exactly what is written.
-        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
-            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
-            drvRequireExperiment(*drv);
-            return DerivedPath::Built {
-                .drvPath = std::move(drv),
-                .outputs = outputSpec,
-            };
-        },
-    }, extendedOutputsSpec.raw);
+        extendedOutputsSpec.raw
+    );
     return InstallableDerivedPath {
         store,
         std::move(derivedPath),

lix/libcmd/installable-flake.cc

@@ -98,7 +98,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths(EvalState & state)
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(std::move(drvPath)),
+            .drvPath = makeConstantStorePath(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;
@@ -136,9 +136,9 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths(EvalState & state)
     }};
 }
 
-std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
+std::pair<Value, PosIdx> InstallableFlake::toValue(EvalState & state)
 {
-    return {&getCursor(state)->forceValue(state), noPos};
+    return {getCursor(state)->forceValue(state), noPos};
 }
 
 std::vector<ref<eval_cache::AttrCursor>>

lix/libcmd/installable-flake.hh

@@ -55,7 +55,7 @@ struct InstallableFlake : InstallableValue
 
     DerivedPathsWithInfo toDerivedPaths(EvalState & state) override;
 
-    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+    std::pair<Value, PosIdx> toValue(EvalState & state) override;
 
     /**
      * Get a cursor to every attrpath in getActualAttrPaths() that

lix/libcmd/installable-value.cc

@@ -9,8 +9,9 @@ std::vector<ref<eval_cache::AttrCursor>>
 InstallableValue::getCursors(EvalState & state)
 {
     auto evalCache =
-        std::make_shared<nix::eval_cache::EvalCache>(std::nullopt,
-            [&](EvalState & state) { return toValue(state).first; });
+        std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, [&](EvalState & state) {
+            return toValue(state).first;
+        });
     return {evalCache->getRoot()};
 }
 

lix/libcmd/installable-value.hh

@@ -77,7 +77,7 @@ struct InstallableValue : Installable
 
     virtual ~InstallableValue() { }
 
-    virtual std::pair<Value *, PosIdx> toValue(EvalState & state) = 0;
+    virtual std::pair<Value, PosIdx> toValue(EvalState & state) = 0;
 
     /**
      * Get a cursor to each value this Installable could refer to.

lix/libcmd/installables.cc

@@ -61,7 +61,7 @@ MixFlakeOptions::MixFlakeOptions()
         .category = category,
         .handler = {[&]() {
             lockFlags.useRegistries = false;
-            warn("'--no-registries' is deprecated; use '--no-use-registries'");
+            printTaggedWarning("'--no-registries' is deprecated; use '--no-use-registries'");
         }}
     });
 
@@ -235,15 +235,14 @@ void SourceExprCommand::completeInstallable(EvalState & state, AddCompletions &
                 prefix_ = "";
             }
 
-            auto [v, pos] = findAlongAttrPath(state, prefix_, *autoArgs, root);
-            Value &v1(*v);
+            auto [v1, pos] = findAlongAttrPath(state, prefix_, *autoArgs, root);
             state.forceValue(v1, pos);
             Value v2;
             state.autoCallFunction(*autoArgs, v1, v2, pos);
 
             if (v2.type() == nAttrs) {
-                for (auto & i : *v2.attrs) {
-                    std::string name = evaluator->symbols[i.name];
+                for (auto & i : *v2.attrs()) {
+                    std::string name{evaluator->symbols[i.name]};
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
                             completions.add(name);
@@ -344,7 +343,7 @@ void completeFlakeRefWithFragment(
             }
         }
     } catch (Error & e) {
-        warn(e.msg());
+        printTaggedWarning("%1%", Uncolored(e.msg()));
     }
 }
 
@@ -412,12 +411,12 @@ ref<eval_cache::EvalCache> openEvalCache(
             if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
                 throw Error("not everything is cached, but evaluation is not allowed");
 
-            auto vFlake = state.ctx.mem.allocValue();
-            flake::callFlake(state, *lockedFlake, *vFlake);
+            Value vFlake;
+            flake::callFlake(state, *lockedFlake, vFlake);
 
-            state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
+            state.forceAttrs(vFlake, noPos, "while parsing cached flake data");
 
-            auto aOutputs = vFlake->attrs->get(state.ctx.symbols.create("outputs"));
+            auto aOutputs = vFlake.attrs()->get(state.ctx.symbols.create("outputs"));
             assert(aOutputs);
 
             return aOutputs->value;
@@ -450,25 +449,24 @@ Installables SourceExprCommand::parseInstallables(
             throw UsageError("'--file' and '--expr' are exclusive");
 
         auto evaluator = getEvaluator();
-        auto vFile = evaluator->mem.allocValue();
+        Value vFile;
 
         if (file == "-") {
             auto & e = evaluator->parseStdin();
-            state.eval(e, *vFile);
+            state.eval(e, vFile);
         }
         else if (file)
-            state.evalFile(state.aio.blockOn(lookupFileArg(*evaluator, *file)).unwrap(), *vFile);
+            state.evalFile(state.aio.blockOn(lookupFileArg(*evaluator, *file)).unwrap(), vFile);
         else {
             auto & e = evaluator->parseExprFromString(*expr, CanonPath::fromCwd());
-            state.eval(e, *vFile);
+            state.eval(e, vFile);
         }
 
         for (auto & s : ss) {
             auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s);
-            result.push_back(
-                make_ref<InstallableAttrPath>(
-                    InstallableAttrPath::parse(
-                        evaluator, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec))));
+            result.push_back(make_ref<InstallableAttrPath>(InstallableAttrPath::parse(
+                evaluator, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec)
+            )));
         }
 
     } else {
@@ -524,36 +522,6 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-static kj::Promise<Result<SingleBuiltPath>> getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
-try {
-    auto handlers = overloaded{
-        [&](const SingleDerivedPath::Opaque & bo) -> kj::Promise<Result<SingleBuiltPath>> {
-            return {SingleBuiltPath::Opaque { bo.path }};
-        },
-        // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-        [&](const SingleDerivedPath::Built & bfd) -> kj::Promise<Result<SingleBuiltPath>> {
-            try {
-                auto drvPath = TRY_AWAIT(getBuiltPath(evalStore, store, *bfd.drvPath));
-                // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work.
-                SingleDerivedPath::Built truncatedBfd {
-                    .drvPath = makeConstantStorePathRef(drvPath.outPath()),
-                    .output = bfd.output,
-                };
-                auto outputPath = TRY_AWAIT(resolveDerivedPath(*store, truncatedBfd, &*evalStore));
-                co_return SingleBuiltPath::Built {
-                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
-                    .output = { bfd.output, outputPath },
-                };
-            } catch (...) {
-                co_return result::current_exception();
-            }
-        },
-    };
-    co_return TRY_AWAIT(std::visit(handlers, b.raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
-
 std::vector<BuiltPathWithResult> Installable::build(
     EvalState & state,
     ref<Store> evalStore,
@@ -642,7 +610,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             state.aio.blockOn(resolveDerivedPath(*store, bfd, &*evalStore));
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
+                                .drvPath = bfd.drvPath,
                                 .outputs = outputs,
                              },
                             .info = aux.info}});
@@ -674,7 +642,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
+                                .drvPath = bfd.drvPath,
                                 .outputs = outputs,
                             },
                             .info = aux.info,
@@ -789,7 +757,7 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(state.aio.blockOn(resolveDerivedPath(*store, *bfd.drvPath)));
+                    drvPaths.insert(bfd.drvPath.path);
                 },
             }, b.path.raw());
 

lix/libcmd/lix-cmd.pc.in

@@ -5,5 +5,6 @@ includedir=@includedir@
 Name: Lix (libcmd)
 Description: Lix Package Manager (libcmd)
 Version: @PACKAGE_VERSION@
-Requires: lix-base lix-util
-Libs: -L${libdir} -llixcmd
+Requires: lix-base lix-util lix-store
+Requires.private: lix-fetchers lix-expr lix-main @BOEHM_IF_FOUND@ libeditline lowdown ncurses
+Libs: -L${libdir} @LIBLIX_DOC_IF_STATIC@ -llixcmd

lix/libcmd/markdown.cc

@@ -3,25 +3,80 @@
 #include "lix/libutil/finally.hh"
 #include "lix/libutil/terminal.hh"
 
+#include <cstdlib>
+#include <iterator>
+#include <new>
+#include <regex>
 #include <sys/queue.h>
 #include <lowdown.h>
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown)
+static const std::string DOCROOT = "@docroot@";
+static const std::string DOCROOT_URL = "https://docs.lix.systems/manual/lix/stable";
+
+static void processLinks(struct lowdown_node * node)
+{
+    if (node->type == LOWDOWN_LINK) {
+        struct lowdown_buf *link = &node->rndr_link.link;
+        if (link && link->size && std::string_view(link->data, link->size).starts_with(DOCROOT)) {
+            // link starts with @docroot@, replace that and check the path extension too.
+            static std::regex mdRewrite{"\\.md(#.*)?$"}; // NOLINT(lix-foreign-exceptions)
+
+            auto oldLink = std::string_view(link->data, link->size).substr(DOCROOT.size());
+            std::string newLink = DOCROOT_URL;
+            std::regex_replace(
+                std::back_inserter(newLink), oldLink.begin(), oldLink.end(), mdRewrite, ".html$1"
+            );
+            if (link->maxsize < newLink.size()) {
+                // the existing link buffer doesn't have enough space for the new string
+                char *newData;
+                if (!(newData = static_cast<char *>(std::realloc(link->data, newLink.size())))) {
+                    throw std::bad_alloc();
+                }
+                link->data = newData;
+                link->maxsize = newLink.size();
+            }
+            newLink.copy(link->data, newLink.size());
+            link->size = newLink.size();
+        }
+    } else {
+        // recurse into children
+        struct lowdown_node *child;
+        TAILQ_FOREACH(child, &node->children, entries)
+            processLinks(child);
+    }
+}
+
+std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno)
 {
     int windowWidth = getWindowSize().second;
+    size_t lowdown_cols = std::max(windowWidth - 5, 60);
 
-    struct lowdown_opts opts {
+    struct lowdown_opts opts{
         .type = LOWDOWN_TERM,
+#ifdef LOWDOWN_SEPARATE_TERM_OPTS
+        .term =
+            {
+                .cols = lowdown_cols,
+                .width = 0,
+                .hmargin = 0,
+                .hpadding = 4,
+                .vmargin = 0,
+                .centre = 0,
+            },
+        // maxdepth needs to be part of the ifdefs to match declaration order
         .maxdepth = 20,
-        .cols = (size_t) std::max(windowWidth - 5, 60),
+#else
+        .maxdepth = 20,
+        .cols = lowdown_cols,
         .hmargin = 0,
         .vmargin = 0,
+#endif /* LOWDOWN_SEPARATE_TERM_OPTS */
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
         .oflags = LOWDOWN_TERM_NOLINK,
     };
-    if (!shouldANSI()) {
+    if (!shouldANSI(fileno)) {
         opts.oflags |= LOWDOWN_TERM_NOANSI;
     }
 
@@ -36,6 +91,8 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
 
+    processLinks(node);
+
     auto renderer = lowdown_term_new(&opts);
     if (!renderer)
         throw Error("cannot allocate Markdown renderer");

lix/libcmd/markdown.hh

@@ -1,10 +1,11 @@
 #pragma once
 ///@file
 
+#include "lix/libutil/terminal.hh"
 #include "lix/libutil/types.hh"
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown);
+std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno = StandardOutputStream::Stdout);
 
 }

lix/libcmd/meson.build

@@ -43,15 +43,16 @@ libcmd = library(
   dependencies : [
     liblixutil,
     liblixstore,
-    liblixexpr,
     liblixfetchers,
+    liblixexpr,
     liblixmain,
-    ncurses,
-    editline,
-    lowdown,
-    nlohmann_json,
     liblix_doc,
+    boehm,
+    editline,
     kj,
+    lowdown,
+    ncurses,
+    nlohmann_json,
   ],
   # '../..' for self references like "lix/libcmd/*.hh"
   include_directories : [ '../..' ],
@@ -72,6 +73,11 @@ custom_target(
 
 liblixcmd = declare_dependency(
   include_directories : include_directories('../..'),
+  dependencies : [
+    liblixutil,
+    liblixstore,
+    kj,
+  ],
   link_with : libcmd,
 )
 meson.override_dependency('lix-cmd', liblixcmd)
@@ -87,5 +93,7 @@ configure_file(
     'libdir' : libdir,
     'includedir' : includedir,
     'PACKAGE_VERSION' : meson.project_version(),
+    'BOEHM_IF_FOUND' : boehm.found() ? 'bdw-gc' : '',
+    'LIBLIX_DOC_IF_STATIC' : is_static ? '-llix_doc' : '',
   },
 )

lix/libcmd/repl-interacter.cc

@@ -244,8 +244,7 @@ void ReadlineLikeInteracter::writeHistory()
     // them so the user isn't confused why their history is getting eaten.
 
     std::string_view const errMsg(std::strerror(writeHistErr));
-    warn("ignoring error writing repl history to %s: %s", this->historyFile, errMsg);
-
+    printTaggedWarning("ignoring error writing repl history to %s: %s", this->historyFile, errMsg);
 }
 
 ReadlineLikeInteracter::~ReadlineLikeInteracter()

lix/libcmd/repl.cc

@@ -3,9 +3,9 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
 #include <string_view>
 
+#include "lix/libexpr/value.hh"
 #include "lix/libutil/box_ptr.hh"
 #include "lix/libcmd/repl-interacter.hh"
 #include "lix/libcmd/repl.hh"
@@ -174,35 +174,37 @@ struct NixRepl
     /**
      * Get a list of each of the `repl-overlays` (parsed and evaluated).
      */
-    Value * replOverlays();
+    Value replOverlays();
 
     /**
      * Get the Nix function that composes the `repl-overlays` together.
      */
-    Value * getReplOverlaysEvalFunction();
+    Value getReplOverlaysEvalFunction();
 
     /**
      * Cached return value of `getReplOverlaysEvalFunction`.
      *
      * Note: This is `shared_ptr` to avoid garbage collection.
      */
-    std::shared_ptr<Value *> replOverlaysEvalFunction =
-        std::allocate_shared<Value *>(TraceableAllocator<Value *>(), nullptr);
+    std::shared_ptr<std::optional<Value>> replOverlaysEvalFunction =
+        std::allocate_shared<std::optional<Value>>(
+            TraceableAllocator<std::optional<Value>>(), std::nullopt
+        );
 
     /**
      * Get the `info` AttrSet that's passed as the first argument to each
      * of the `repl-overlays`.
      */
-    Value * replInitInfo();
+    Value replInitInfo();
 
     /**
      * Get the current top-level bindings as an AttrSet.
      */
-    Value * bindingsToAttrs();
+    Value bindingsToAttrs();
     /**
      * Parse a file, evaluate its result, and force the resulting value.
      */
-    Value * evalFile(SourcePath & path);
+    Value evalFile(SourcePath & path);
 
     void printValue(std::ostream & str,
                               Value & v,
@@ -254,7 +256,7 @@ void runNix(Path program, const Strings & args)
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
-    }).wait();
+    }).waitAndCheck();
 
     return;
 }
@@ -291,7 +293,7 @@ ReplExitStatus NixRepl::mainLoop()
         if (evaluator.debug && evaluator.debug->inDebugger) {
             debuggerNotice = " debugger";
         }
-        notice("Lix %1%%2%\nType :? for help.", nixVersion, debuggerNotice);
+        notice("Lix %1%%2%\nType :? for help.", Uncolored(nixVersion), debuggerNotice);
     }
 
     isFirstRepl = false;
@@ -307,7 +309,7 @@ ReplExitStatus NixRepl::mainLoop()
     std::string input;
 
     while (true) {
-        _isInterrupted = false;
+        unsetUserInterruptRequest();
 
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
@@ -338,14 +340,14 @@ ReplExitStatus NixRepl::mainLoop()
                 // input without clearing the input so far.
                 continue;
             } else {
-              printMsg(lvlError, e.msg());
+                printMsg(lvlError, "%1%", Uncolored(e.msg()));
             }
         } catch (EvalError & e) {
-            printMsg(lvlError, e.msg());
+            printMsg(lvlError, "%1%", Uncolored(e.msg()));
         } catch (Error & e) {
-            printMsg(lvlError, e.msg());
+            printMsg(lvlError, "%1%", Uncolored(e.msg()));
         } catch (Interrupted & e) {
-            printMsg(lvlError, e.msg());
+            printMsg(lvlError, "%1%", Uncolored(e.msg()));
         }
 
         // We handled the current input fully, so we should clear it
@@ -451,7 +453,7 @@ StringSet NixRepl::completePrefix(const std::string &prefix)
             e.eval(state, *env, v);
             state.forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
-            for (auto & i : *v.attrs) {
+            for (auto & i : *v.attrs()) {
                 std::ostringstream output;
                 printAttributeName(output, evaluator.symbols[i.name]);
                 std::string name = output.str();
@@ -654,7 +656,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 auto path = state.coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
-                auto pos = evaluator.positions[v.lambda.fun->pos];
+                auto pos = evaluator.positions[v.lambda().fun->pos];
                 if (auto path = std::get_if<CheckedSourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
@@ -672,7 +674,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait();
+        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).waitAndCheck();
 
         // Reload right after exiting the editor if path is not in store
         // Store is immutable, so there could be no changes, so there's no need to reload
@@ -760,7 +762,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
             state.aio.blockOn(evaluator.store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(drvPath),
+                    .drvPath = makeConstantStorePath(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             }));
@@ -789,7 +791,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
         Value v;
         evalString(arg, v);
         if (v.type() == nString) {
-            std::cout << v.string.s;
+            std::cout << v.str();
         } else {
             printValue(std::cout, v);
         }
@@ -823,7 +825,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
         } else if (v.isLambda()) {
-            auto pos = evaluator.positions[v.lambda.fun->pos];
+            auto pos = evaluator.positions[v.lambda().fun->pos];
             if (auto path = std::get_if<CheckedSourcePath>(&pos.origin)) {
                 // Path and position have now been obtained, feed to nix-doc library to get data.
                 auto docComment = lambdaDocsForPos(*path, pos);
@@ -863,10 +865,10 @@ ProcessLineResult NixRepl::processLine(std::string line)
         std::visit(overloaded {
             [&](ExprReplBindings & b) {
                 for (auto & [name, e] : b.symbols) {
-                    Value * v = state.ctx.mem.allocValue();
-                    e->eval(state, *env, *v);
+                    Value v;
+                    e->eval(state, *env, v);
                     (void) e.release(); // NOLINT(bugprone-unused-return-value): leak because of thunk references
-                    addVarToScope(name, *v);
+                    addVarToScope(name, v);
                 }
             },
             [&](std::unique_ptr<Expr> & e) {
@@ -949,7 +951,7 @@ void NixRepl::loadFiles()
 
     for (auto & [i, what] : getValues()) {
         notice("Loading installable '%1%'...", Magenta(what));
-        addAttrsToScope(*i);
+        addAttrsToScope(i);
     }
 
     loadReplOverlays();
@@ -961,12 +963,12 @@ void NixRepl::loadReplOverlays()
         return;
     }
 
-    notice("Loading '%1%'...", Magenta("repl-overlays"));
+    notice("Loading '%1%'...", "repl-overlays");
     auto replInitFilesFunction = getReplOverlaysEvalFunction();
 
-    Value &newAttrs(*evaluator.mem.allocValue());
-    SmallValueVector<3> args = {replInitInfo(), bindingsToAttrs(), replOverlays()};
-    state.callFunction(*replInitFilesFunction, args.size(), args.data(), newAttrs, noPos);
+    Value newAttrs;
+    Value args[] = {replInitInfo(), bindingsToAttrs(), replOverlays()};
+    state.callFunction(replInitFilesFunction, args, newAttrs, noPos);
 
     // n.b. this does in fact load the stuff into the environment twice (once
     // from the superset of the environment returned by repl-overlays and once
@@ -976,14 +978,14 @@ void NixRepl::loadReplOverlays()
     addAttrsToScope(newAttrs);
 }
 
-Value * NixRepl::getReplOverlaysEvalFunction()
+Value NixRepl::getReplOverlaysEvalFunction()
 {
     if (replOverlaysEvalFunction && *replOverlaysEvalFunction) {
-        return *replOverlaysEvalFunction;
+        return **replOverlaysEvalFunction;
     }
 
     auto evalReplInitFilesPath = CanonPath::root + "repl-overlays.nix";
-    *replOverlaysEvalFunction = evaluator.mem.allocValue();
+    *replOverlaysEvalFunction = Value{};
     auto code =
         #include "repl-overlays.nix.gen.hh"
         ;
@@ -995,42 +997,55 @@ Value * NixRepl::getReplOverlaysEvalFunction()
 
     state.eval(expr, **replOverlaysEvalFunction);
 
-    return *replOverlaysEvalFunction;
+    return **replOverlaysEvalFunction;
 }
 
-Value * NixRepl::replOverlays()
+Value NixRepl::replOverlays()
 {
-    Value * replInits(evaluator.mem.allocValue());
-    *replInits = evaluator.mem.newList(evalSettings.replOverlays.get().size());
-    Value ** replInitElems = replInits->listElems();
+    Value replInits;
+    auto replInitElems = evaluator.mem.newList(evalSettings.replOverlays.get().size());
+    replInits = {NewValueAs::list, replInitElems};
 
     size_t i = 0;
     for (auto path : evalSettings.replOverlays.get()) {
         debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
         SourcePath sourcePath((CanonPath(path)));
+
+        // XXX(jade): This is a somewhat unsatisfying solution to
+        // https://git.lix.systems/lix-project/lix/issues/777 which means that
+        // the top level item in the repl-overlays file (that is, the lambda)
+        // gets evaluated with pure eval off. This means that if you want to do
+        // impure eval stuff, you will have to force it with builtins.seq.
+        bool prevPureEval = evalSettings.pureEval.get();
         auto replInit = evalFile(sourcePath);
+        evalSettings.pureEval.setDefault(prevPureEval);
 
-        if (!replInit->isLambda()) {
-            evaluator.errors.make<TypeError>(
-                "Expected `repl-overlays` entry %s to be a lambda but found %s: %s",
-                path,
-                showType(*replInit),
-                ValuePrinter(state, *replInit, errorPrintOptions)
-            )
-            .debugThrow();
-        }
-
-        if (auto attrs = dynamic_cast<AttrsPattern *>(replInit->lambda.fun->pattern.get()); attrs && !attrs->ellipsis) {
-            evaluator.errors.make<TypeError>(
-                "Expected first argument of %1% to have %2% to allow future versions of Lix to add additional attributes to the argument",
-                "repl-overlays",
-                "..."
-            )
-                .atPos(replInit->lambda.fun->pos)
+        if (!replInit.isLambda()) {
+            evaluator.errors
+                .make<TypeError>(
+                    "Expected `repl-overlays` entry %s to be a lambda but found %s: %s",
+                    path,
+                    showType(replInit),
+                    ValuePrinter(state, replInit, errorPrintOptions)
+                )
                 .debugThrow();
         }
 
-        replInitElems[i] = replInit;
+        if (auto attrs = dynamic_cast<AttrsPattern *>(replInit.lambda().fun->pattern.get());
+            attrs && !attrs->ellipsis)
+        {
+            evaluator.errors
+                .make<TypeError>(
+                    "Expected first argument of %1% to have %2% to allow future versions of Lix to "
+                    "add additional attributes to the argument",
+                    "repl-overlays",
+                    "..."
+                )
+                .atPos(replInit.lambda().fun->pos)
+                .debugThrow();
+        }
+
+        replInitElems->elems[i] = replInit;
         i++;
     }
 
@@ -1038,16 +1053,16 @@ Value * NixRepl::replOverlays()
     return replInits;
 }
 
-Value * NixRepl::replInitInfo()
+Value NixRepl::replInitInfo()
 {
     auto builder = evaluator.buildBindings(2);
 
-    Value * currentSystem(evaluator.mem.allocValue());
-    currentSystem->mkString(evalSettings.getCurrentSystem());
+    Value currentSystem;
+    currentSystem.mkString(evalSettings.getCurrentSystem());
     builder.insert(evaluator.symbols.create("currentSystem"), currentSystem);
 
-    Value * info(evaluator.mem.allocValue());
-    info->mkAttrs(builder.finish());
+    Value info;
+    info.mkAttrs(builder.finish());
     return info;
 }
 
@@ -1056,19 +1071,24 @@ template<typename T, typename NameFn, typename ValueFn>
 void NixRepl::addToScope(T && things, NameFn nameFn, ValueFn valueFn)
 {
     size_t added = 0;
-    for (auto && thing : things) {
-        if (displ + 1 >= envSize)
-            throw Error("environment full; cannot add more variables");
 
-        const auto name = nameFn(thing);
-        staticEnv->vars.emplace_back(name, displ);
-        env->values[displ++] = valueFn(thing);
-        varNames.emplace(evaluator.symbols[name]);
-        added++;
-    }
+    staticEnv->vars.unsafe_insert_bulk([&] (auto & map) {
+        auto oldSize = map.size();
+        for (auto && thing : things) {
+            if (displ + 1 >= envSize)
+                throw Error("environment full; cannot add more variables");
+
+            const auto name = nameFn(thing);
+            map.emplace_back(name, displ);
+            env->values[displ++] = valueFn(thing);
+            varNames.emplace(evaluator.symbols[name]);
+            added++;
+        }
+        // safety: we sort the range that we inserted so that we don't have to push that
+        // invariant up to the caller
+        std::sort(map.begin() + oldSize, map.end());
+    });
 
-    staticEnv->sort();
-    staticEnv->deduplicate();
     if (added > 0) {
         notice("Added %1% variables.", added);
     }
@@ -1077,7 +1097,9 @@ void NixRepl::addToScope(T && things, NameFn nameFn, ValueFn valueFn)
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state.forceAttrs(attrs, noPos, "while evaluating an attribute set to be merged in the global scope");
-    addToScope(*attrs.attrs, [](Attr & a) { return a.name; }, [](Attr & a) { return a.value; });
+    addToScope(
+        *attrs.attrs(), [](const Attr & a) { return a.name; }, [](const Attr & a) { return a.value; }
+    );
 }
 
 void NixRepl::addValMapToScope(const ValMap & attrs)
@@ -1093,27 +1115,24 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 {
     if (displ >= envSize)
         throw Error("environment full; cannot add more variables");
-    if (auto oldVar = staticEnv->find(name); oldVar != staticEnv->vars.end()) {
-        staticEnv->vars.erase(oldVar);
+    if (staticEnv->vars.insert_or_assign(name, displ).second) {
         notice("Updated %s.", evaluator.symbols[name]);
     } else {
         notice("Added %s.", evaluator.symbols[name]);
     }
-    staticEnv->vars.emplace_back(name, displ);
-    staticEnv->sort();
-    env->values[displ++] = &v;
+    env->values[displ++] = v;
     varNames.emplace(evaluator.symbols[name]);
 }
 
-Value * NixRepl::bindingsToAttrs()
+Value NixRepl::bindingsToAttrs()
 {
     auto builder = evaluator.buildBindings(staticEnv->vars.size());
     for (auto & [symbol, displacement] : staticEnv->vars) {
         builder.insert(symbol, env->values[displacement]);
     }
 
-    Value * attrs(evaluator.mem.allocValue());
-    attrs->mkAttrs(builder.finish());
+    Value attrs;
+    attrs.mkAttrs(builder.finish());
     return attrs;
 }
 
@@ -1136,12 +1155,12 @@ void NixRepl::evalString(std::string s, Value & v)
     state.forceValue(v, noPos);
 }
 
-Value * NixRepl::evalFile(SourcePath & path)
+Value NixRepl::evalFile(SourcePath & path)
 {
     auto & expr = evaluator.parseExprFromFile(evaluator.paths.checkSourcePath(path), staticEnv);
-    Value * result(evaluator.mem.allocValue());
-    expr.eval(state, *env, *result);
-    state.forceValue(*result, noPos);
+    Value result;
+    expr.eval(state, *env, result);
+    state.forceValue(result, noPos);
     return result;
 }