From 01ff67595bb138b8d1db866d2c5cedd17627f16c Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Mon, 16 Feb 2026 10:24:19 -0800 Subject: [PATCH] nix3-add-path: add references-list argument for ca with references This is not ca-derivations. We don't care about self-references or any of the rewriting or stuff like that; if you want something like that, write your user code so it figures out where it is. The reason I want to do this is for integrating other build systems with Lix: if you're importing something built *outside the store* with a non-Nix build system, it makes no sense to put it in an input-addressed path since it's not possible to come up with a derivation for it. But you need *some* hash, so the output hashing is the most sensible option. It is also nice because of not needing trusted user privileges to import it, since the hash describes the exact contents of the output path. We use this feature (implemented externally to Lix; this CL eliminates that step) at Mercury to integrate buck2 with Nix on the output side: we import things to the Nix store as ca paths with references. These can then be consumed by Nix language with: ``` # Hack from https://git.lix.systems/lix-project/lix/issues/402#issuecomment-5889 path: builtins.appendContext path { ${path} = { path = true; }; } ``` Test plan: ``` $ nix store add-path --references-list xx-refs-list ./README.md /nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md $ cat xx-refs-list /nix/store/g9hhwjbkdrw0wnbd3axfs4icyb37nr6b-bash-interactive-5.3p3 $ nix path-info --json /nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md | jq . [ { "ca": "fixed:r:sha256:1pj59fy7zcrn949ry9kgv4ba17h2sy3z2is469abj912cvbl14n3", "narHash": "sha256-w5JA12YiJLlUMkRH8YfXAp6gFtlvJp8TSTazf7xLRd4=", "narSize": 1384, "path": "/nix/store/szcwnm13d9gmxx8fly0bz82l42jgysw6-README.md", "references": [ "/nix/store/g9hhwjbkdrw0wnbd3axfs4icyb37nr6b-bash-interactive-5.3p3" ], "registrationTime": 1771266146, "valid": true } ] ``` Change-Id: I005a03003dfc24108e018e599dbe0b5d6a6a6964 --- doc/manual/rl-next/add-path-references.md | 25 +++++++++ lix/nix/add-to-store.cc | 28 ++++++++-- tests/functional2/store/test_add.py | 66 +++++++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 doc/manual/rl-next/add-path-references.md diff --git a/doc/manual/rl-next/add-path-references.md b/doc/manual/rl-next/add-path-references.md new file mode 100644 index 000000000..26aa93b21 --- /dev/null +++ b/doc/manual/rl-next/add-path-references.md @@ -0,0 +1,25 @@ +--- +synopsis: "nix store add-path now supports references" +cls: [5205] +category: "Features" +credits: [jade] +--- +Lix supports two categories of hashes in store paths: input-addressed and output-addressed. + +Currently, in Nix language, there is no way to produce output-addressed paths with references, as fixed-output derivations forbid references. +However, the Nix store actually *supports* references in output-addressed paths. +This is very useful for importing build products created outside of Lix that reference dependency store paths since such build products have no associated derivation so don't make any sense to input-address. +Previously, output-addressed paths with references could only be created by writing a custom client to the rather-baroque Nix daemon protocol; now it's available in the CLI. + +Using `nix store add-path --references-list-json REFS_LIST_FILE SOME_PATH` with a JSON list of string store paths, you can now create such paths with the Lix CLI. +They may be consumed from Nix language using something like `builtins.storePath` or the following which also works in pure evaluation mode: + +```nix +# Hack from https://git.lix.systems/lix-project/lix/issues/402#issuecomment-5889 +path: +builtins.appendContext path { + ${path} = { + path = true; + }; +} +``` diff --git a/lix/nix/add-to-store.cc b/lix/nix/add-to-store.cc index 4a082fdb1..355ac4d0c 100644 --- a/lix/nix/add-to-store.cc +++ b/lix/nix/add-to-store.cc @@ -1,8 +1,11 @@ #include "lix/libcmd/command.hh" #include "lix/libmain/common-args.hh" +#include "lix/libstore/path.hh" #include "lix/libstore/store-api.hh" #include "lix/libutil/archive.hh" #include "lix/libutil/async-io.hh" +#include "lix/libutil/file-system.hh" +#include "lix/libutil/json.hh" #include "add-to-store.hh" namespace nix { @@ -11,6 +14,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + std::optional referencesListFile; FileIngestionMethod ingestionMethod; CmdAddToStore() @@ -29,7 +33,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand void run(ref store) override { + StorePathSet references{}; if (!namePart) namePart = baseNameOf(path); + if (referencesListFile.has_value()) { + auto parsed = json::parse(readFile(*referencesListFile), "references list file"); + for (auto it : parsed.get>()) { + references.insert(store->parseStorePath(it)); + } + } StringSink sink; sink << dumpPath(path); @@ -43,13 +54,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand hash = hsink.finish().first; } - ValidPathInfo info { + ValidPathInfo info{ *store, std::move(*namePart), - FixedOutputInfo { + FixedOutputInfo{ .method = std::move(ingestionMethod), .hash = std::move(hash), - .references = {}, + .references = {references}, }, narHash, }; @@ -89,6 +100,17 @@ struct CmdAddPath : CmdAddToStore CmdAddPath() { ingestionMethod = FileIngestionMethod::Recursive; + + // References are only available for the recursive ingest method; the + // store will tell us "fixed output derivation is not allowed to refer + // to other store paths" for the flat ingest method. + addFlag({ + .longName = "references-list-json", + .description = "File containing a JSON list of references of the to-be-added store path", + .labels = {"file"}, + .handler = {&referencesListFile}, + .completer = completePath, + }); } std::string description() override diff --git a/tests/functional2/store/test_add.py b/tests/functional2/store/test_add.py index 1884c2333..9579db7cb 100644 --- a/tests/functional2/store/test_add.py +++ b/tests/functional2/store/test_add.py @@ -1,7 +1,13 @@ +import json +from pathlib import Path +from typing import Concatenate +from collections.abc import Callable + import pytest from testlib.fixtures.file_helper import with_files, File, AssetSymlink, Symlink from testlib.fixtures.nix import Nix +from testlib.fixtures.command import Command @with_files({"file-link": AssetSymlink("./test_add.py"), "dir-link": Symlink(".")}) @@ -48,3 +54,63 @@ def test_hash(nix: Nix, blank_add_path: str): res = nix.nix(["--type", "sha256", "--base32", "./dummy"], "nix-hash").run().ok() hash2 = f"sha256:{res.stdout_plain}" assert hash1 == hash2 + + +@with_files({"dummy": File("Hello World\n"), "item2": File("foo bar")}) +class TestNix3AddPath: + @pytest.fixture(autouse=True) + def enable_flakes(self, nix: Nix): + nix.settings.add_xp_feature("nix-command", "flakes") + + # type checkers hate this one weird trick + AddPath = Callable[Concatenate[str, ...], Command] + + @pytest.fixture + def add_path(self, nix: Nix) -> AddPath: + def inner(name: str, references_list: str | None = None) -> Command: + references_list_arg = ( + ["--references-list-json", references_list] if references_list else [] + ) + return nix.nix(["store", "add-path", *references_list_arg, name]) + + return inner + + def test_nix3_rec_basic(self, add_path: AddPath, blank_add_path: str): + res = add_path("./dummy").run().ok().stdout_plain + assert res == blank_add_path + + def test_nix3_rec_empty_references(self, files: Path, add_path: AddPath, blank_add_path: str): + # no references + (files / "reflist.json").write_text("[]") + + path = add_path("./dummy", "reflist.json").run().ok().stdout_plain + assert path == blank_add_path + + def test_nix3_rec_bad_json(self, files: Path, add_path: AddPath): + (files / "reflist.json").write_text("parse error") + + err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain + assert "references list file" in err + + def test_nix3_rec_some_references( + self, files: Path, nix: Nix, add_path: AddPath, blank_add_path: str + ): + path2 = add_path("item2").run().ok().stdout_plain + + # some references, which should cause a different output path + (files / "reflist.json").write_text(json.dumps([blank_add_path, path2])) + + path = add_path("./dummy", "reflist.json").run().ok().stdout_plain + assert path != blank_add_path + + # and the resulting path in the store also has the right references + out = nix.nix(["path-info", "--json", path]).run().ok().stdout_s + out = json.loads(out) + assert set(out[0]["references"]) == {blank_add_path, path2} + + def test_nix3_rec_bad_json_type(self, files: Path, add_path: AddPath): + (files / "reflist.json").write_text("{}") + + err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain + + assert "type must be array" in err