nix3-add-path: add references-list argument for ca with references

This is not ca-derivations. We don't care about self-references or any of the
rewriting or stuff like that; if you want something like that, write
your user code so it figures out where it is.

The reason I want to do this is for integrating other build systems with
Lix: if you're importing something built *outside the store* with a
non-Nix build system, it makes no sense to put it in an input-addressed
path since it's not possible to come up with a derivation for it.
But you need *some* hash, so the output hashing is the most sensible option.
It is also nice because of not needing trusted user privileges to import
it, since the hash describes the exact contents of the output path.

We use this feature (implemented externally to Lix; this CL eliminates
that step) at Mercury to integrate buck2 with Nix on the output side: we
import things to the Nix store as ca paths with references.

These can then be consumed by Nix language with:

```
# Hack from https://git.lix.systems/lix-project/lix/issues/402#issuecomment-5889
path:
builtins.appendContext path {
  ${path} = {
    path = true;
  };
}
```

Test plan:
```
$ nix store add-path --references-list xx-refs-list ./README.md
/nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md

$ cat xx-refs-list
/nix/store/g9hhwjbkdrw0wnbd3axfs4icyb37nr6b-bash-interactive-5.3p3

$ nix path-info --json /nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md | jq .
[
  {
    "ca": "fixed:r:sha256:1pj59fy7zcrn949ry9kgv4ba17h2sy3z2is469abj912cvbl14n3",
    "narHash": "sha256-w5JA12YiJLlUMkRH8YfXAp6gFtlvJp8TSTazf7xLRd4=",
    "narSize": 1384,
    "path": "/nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md",
    "references": [
      "/nix/store/g9hhwjbkdrw0wnbd3axfs4icyb37nr6b-bash-interactive-5.3p3"
    ],
    "registrationTime": 1771266146,
    "valid": true
  }
]
```

Change-Id: I005a03003dfc24108e018e599dbe0b5d6a6a6964

doc/manual/rl-next/add-path-references.md

@@ -0,0 +1,25 @@
+---
+synopsis: "nix store add-path now supports references"
+cls: [5205]
+category: "Features"
+credits: [jade]
+---
+Lix supports two categories of hashes in store paths: input-addressed and output-addressed.
+
+Currently, in Nix language, there is no way to produce output-addressed paths with references, as fixed-output derivations forbid references.
+However, the Nix store actually *supports* references in output-addressed paths.
+This is very useful for importing build products created outside of Lix that reference dependency store paths since such build products have no associated derivation so don't make any sense to input-address.
+Previously, output-addressed paths with references could only be created by writing a custom client to the rather-baroque Nix daemon protocol; now it's available in the CLI.
+
+Using `nix store add-path --references-list-json REFS_LIST_FILE SOME_PATH` with a JSON list of string store paths, you can now create such paths with the Lix CLI.
+They may be consumed from Nix language using something like `builtins.storePath` or the following which also works in pure evaluation mode:
+
+```nix
+# Hack from https://git.lix.systems/lix-project/lix/issues/402#issuecomment-5889
+path:
+builtins.appendContext path {
+  ${path} = {
+    path = true;
+  };
+}
+```

lix/nix/add-to-store.cc

@@ -1,8 +1,11 @@
 #include "lix/libcmd/command.hh"
 #include "lix/libmain/common-args.hh"
+#include "lix/libstore/path.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libutil/archive.hh"
 #include "lix/libutil/async-io.hh"
+#include "lix/libutil/file-system.hh"
+#include "lix/libutil/json.hh"
 #include "add-to-store.hh"
 
 namespace nix {
@@ -11,6 +14,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
     std::optional<std::string> namePart;
+    std::optional<Path> referencesListFile;
     FileIngestionMethod ingestionMethod;
 
     CmdAddToStore()
@@ -29,7 +33,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
     void run(ref<Store> store) override
     {
+        StorePathSet references{};
         if (!namePart) namePart = baseNameOf(path);
+        if (referencesListFile.has_value()) {
+            auto parsed = json::parse(readFile(*referencesListFile), "references list file");
+            for (auto it : parsed.get<std::vector<std::string_view>>()) {
+                references.insert(store->parseStorePath(it));
+            }
+        }
 
         StringSink sink;
         sink << dumpPath(path);
@@ -43,13 +54,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             hash = hsink.finish().first;
         }
 
-        ValidPathInfo info {
+        ValidPathInfo info{
             *store,
             std::move(*namePart),
-            FixedOutputInfo {
+            FixedOutputInfo{
                 .method = std::move(ingestionMethod),
                 .hash = std::move(hash),
-                .references = {},
+                .references = {references},
             },
             narHash,
         };
@@ -89,6 +100,17 @@ struct CmdAddPath : CmdAddToStore
     CmdAddPath()
     {
         ingestionMethod = FileIngestionMethod::Recursive;
+
+        // References are only available for the recursive ingest method; the
+        // store will tell us "fixed output derivation is not allowed to refer
+        // to other store paths" for the flat ingest method.
+        addFlag({
+            .longName = "references-list-json",
+            .description = "File containing a JSON list of references of the to-be-added store path",
+            .labels = {"file"},
+            .handler = {&referencesListFile},
+            .completer = completePath,
+        });
     }
 
     std::string description() override

tests/functional2/store/test_add.py

@@ -1,7 +1,13 @@
+import json
+from pathlib import Path
+from typing import Concatenate
+from collections.abc import Callable
+
 import pytest
 
 from testlib.fixtures.file_helper import with_files, File, AssetSymlink, Symlink
 from testlib.fixtures.nix import Nix
+from testlib.fixtures.command import Command
 
 
 @with_files({"file-link": AssetSymlink("./test_add.py"), "dir-link": Symlink(".")})
@@ -48,3 +54,63 @@ def test_hash(nix: Nix, blank_add_path: str):
     res = nix.nix(["--type", "sha256", "--base32", "./dummy"], "nix-hash").run().ok()
     hash2 = f"sha256:{res.stdout_plain}"
     assert hash1 == hash2
+
+
+@with_files({"dummy": File("Hello World\n"), "item2": File("foo bar")})
+class TestNix3AddPath:
+    @pytest.fixture(autouse=True)
+    def enable_flakes(self, nix: Nix):
+        nix.settings.add_xp_feature("nix-command", "flakes")
+
+    # type checkers hate this one weird trick
+    AddPath = Callable[Concatenate[str, ...], Command]
+
+    @pytest.fixture
+    def add_path(self, nix: Nix) -> AddPath:
+        def inner(name: str, references_list: str | None = None) -> Command:
+            references_list_arg = (
+                ["--references-list-json", references_list] if references_list else []
+            )
+            return nix.nix(["store", "add-path", *references_list_arg, name])
+
+        return inner
+
+    def test_nix3_rec_basic(self, add_path: AddPath, blank_add_path: str):
+        res = add_path("./dummy").run().ok().stdout_plain
+        assert res == blank_add_path
+
+    def test_nix3_rec_empty_references(self, files: Path, add_path: AddPath, blank_add_path: str):
+        # no references
+        (files / "reflist.json").write_text("[]")
+
+        path = add_path("./dummy", "reflist.json").run().ok().stdout_plain
+        assert path == blank_add_path
+
+    def test_nix3_rec_bad_json(self, files: Path, add_path: AddPath):
+        (files / "reflist.json").write_text("parse error")
+
+        err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain
+        assert "references list file" in err
+
+    def test_nix3_rec_some_references(
+        self, files: Path, nix: Nix, add_path: AddPath, blank_add_path: str
+    ):
+        path2 = add_path("item2").run().ok().stdout_plain
+
+        # some references, which should cause a different output path
+        (files / "reflist.json").write_text(json.dumps([blank_add_path, path2]))
+
+        path = add_path("./dummy", "reflist.json").run().ok().stdout_plain
+        assert path != blank_add_path
+
+        # and the resulting path in the store also has the right references
+        out = nix.nix(["path-info", "--json", path]).run().ok().stdout_s
+        out = json.loads(out)
+        assert set(out[0]["references"]) == {blank_add_path, path2}
+
+    def test_nix3_rec_bad_json_type(self, files: Path, add_path: AddPath):
+        (files / "reflist.json").write_text("{}")
+
+        err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain
+
+        assert "type must be array" in err