diff --git a/doc/manual/rl-next/add-path-references.md b/doc/manual/rl-next/add-path-references.md new file mode 100644 index 000000000..26aa93b21 --- /dev/null +++ b/doc/manual/rl-next/add-path-references.md @@ -0,0 +1,25 @@ +--- +synopsis: "nix store add-path now supports references" +cls: [5205] +category: "Features" +credits: [jade] +--- +Lix supports two categories of hashes in store paths: input-addressed and output-addressed. + +Currently, in Nix language, there is no way to produce output-addressed paths with references, as fixed-output derivations forbid references. +However, the Nix store actually *supports* references in output-addressed paths. +This is very useful for importing build products created outside of Lix that reference dependency store paths since such build products have no associated derivation so don't make any sense to input-address. +Previously, output-addressed paths with references could only be created by writing a custom client to the rather-baroque Nix daemon protocol; now it's available in the CLI. + +Using `nix store add-path --references-list-json REFS_LIST_FILE SOME_PATH` with a JSON list of string store paths, you can now create such paths with the Lix CLI. +They may be consumed from Nix language using something like `builtins.storePath` or the following which also works in pure evaluation mode: + +```nix +# Hack from https://git.lix.systems/lix-project/lix/issues/402#issuecomment-5889 +path: +builtins.appendContext path { + ${path} = { + path = true; + }; +} +``` diff --git a/lix/nix/add-to-store.cc b/lix/nix/add-to-store.cc index 4a082fdb1..355ac4d0c 100644 --- a/lix/nix/add-to-store.cc +++ b/lix/nix/add-to-store.cc @@ -1,8 +1,11 @@ #include "lix/libcmd/command.hh" #include "lix/libmain/common-args.hh" +#include "lix/libstore/path.hh" #include "lix/libstore/store-api.hh" #include "lix/libutil/archive.hh" #include "lix/libutil/async-io.hh" +#include "lix/libutil/file-system.hh" +#include "lix/libutil/json.hh" #include "add-to-store.hh" namespace nix { @@ -11,6 +14,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + std::optional referencesListFile; FileIngestionMethod ingestionMethod; CmdAddToStore() @@ -29,7 +33,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand void run(ref store) override { + StorePathSet references{}; if (!namePart) namePart = baseNameOf(path); + if (referencesListFile.has_value()) { + auto parsed = json::parse(readFile(*referencesListFile), "references list file"); + for (auto it : parsed.get>()) { + references.insert(store->parseStorePath(it)); + } + } StringSink sink; sink << dumpPath(path); @@ -43,13 +54,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand hash = hsink.finish().first; } - ValidPathInfo info { + ValidPathInfo info{ *store, std::move(*namePart), - FixedOutputInfo { + FixedOutputInfo{ .method = std::move(ingestionMethod), .hash = std::move(hash), - .references = {}, + .references = {references}, }, narHash, }; @@ -89,6 +100,17 @@ struct CmdAddPath : CmdAddToStore CmdAddPath() { ingestionMethod = FileIngestionMethod::Recursive; + + // References are only available for the recursive ingest method; the + // store will tell us "fixed output derivation is not allowed to refer + // to other store paths" for the flat ingest method. + addFlag({ + .longName = "references-list-json", + .description = "File containing a JSON list of references of the to-be-added store path", + .labels = {"file"}, + .handler = {&referencesListFile}, + .completer = completePath, + }); } std::string description() override diff --git a/tests/functional2/store/test_add.py b/tests/functional2/store/test_add.py index 1884c2333..9579db7cb 100644 --- a/tests/functional2/store/test_add.py +++ b/tests/functional2/store/test_add.py @@ -1,7 +1,13 @@ +import json +from pathlib import Path +from typing import Concatenate +from collections.abc import Callable + import pytest from testlib.fixtures.file_helper import with_files, File, AssetSymlink, Symlink from testlib.fixtures.nix import Nix +from testlib.fixtures.command import Command @with_files({"file-link": AssetSymlink("./test_add.py"), "dir-link": Symlink(".")}) @@ -48,3 +54,63 @@ def test_hash(nix: Nix, blank_add_path: str): res = nix.nix(["--type", "sha256", "--base32", "./dummy"], "nix-hash").run().ok() hash2 = f"sha256:{res.stdout_plain}" assert hash1 == hash2 + + +@with_files({"dummy": File("Hello World\n"), "item2": File("foo bar")}) +class TestNix3AddPath: + @pytest.fixture(autouse=True) + def enable_flakes(self, nix: Nix): + nix.settings.add_xp_feature("nix-command", "flakes") + + # type checkers hate this one weird trick + AddPath = Callable[Concatenate[str, ...], Command] + + @pytest.fixture + def add_path(self, nix: Nix) -> AddPath: + def inner(name: str, references_list: str | None = None) -> Command: + references_list_arg = ( + ["--references-list-json", references_list] if references_list else [] + ) + return nix.nix(["store", "add-path", *references_list_arg, name]) + + return inner + + def test_nix3_rec_basic(self, add_path: AddPath, blank_add_path: str): + res = add_path("./dummy").run().ok().stdout_plain + assert res == blank_add_path + + def test_nix3_rec_empty_references(self, files: Path, add_path: AddPath, blank_add_path: str): + # no references + (files / "reflist.json").write_text("[]") + + path = add_path("./dummy", "reflist.json").run().ok().stdout_plain + assert path == blank_add_path + + def test_nix3_rec_bad_json(self, files: Path, add_path: AddPath): + (files / "reflist.json").write_text("parse error") + + err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain + assert "references list file" in err + + def test_nix3_rec_some_references( + self, files: Path, nix: Nix, add_path: AddPath, blank_add_path: str + ): + path2 = add_path("item2").run().ok().stdout_plain + + # some references, which should cause a different output path + (files / "reflist.json").write_text(json.dumps([blank_add_path, path2])) + + path = add_path("./dummy", "reflist.json").run().ok().stdout_plain + assert path != blank_add_path + + # and the resulting path in the store also has the right references + out = nix.nix(["path-info", "--json", path]).run().ok().stdout_s + out = json.loads(out) + assert set(out[0]["references"]) == {blank_add_path, path2} + + def test_nix3_rec_bad_json_type(self, files: Path, add_path: AddPath): + (files / "reflist.json").write_text("{}") + + err = add_path("./dummy", "reflist.json").run().expect(1).stderr_plain + + assert "type must be array" in err