From 17e6497ca4ee118e10f0a52d9fb5e504d0924168 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 27 Feb 2025 16:08:50 +0100 Subject: [PATCH] libutil: attach nar member names to dentries, not inodes this is where they should've been from the start, but during the first rewrite it made little sense to move them. we have bigger plans today, so we'll finally clean that up too. note the `Map` transform type that is needed to make the current macros work. it shall be only temporary. Change-Id: I928d197dbfe27b68cf8634d149c3259e86fbf123 --- lix/libutil/archive.cc | 105 +++++++++++++------------ lix/libutil/archive.hh | 5 +- tests/unit/libutil/archive.cc | 139 ++++++++++++++++++---------------- 3 files changed, 131 insertions(+), 118 deletions(-) diff --git a/lix/libutil/archive.cc b/lix/libutil/archive.cc index abfb4859d..02a28c216 100644 --- a/lix/libutil/archive.cc +++ b/lix/libutil/archive.cc @@ -167,12 +167,12 @@ struct CaseInsensitiveCompare namespace nar { -static Generator parseObject(Source & source, const Path & path) +static Generator parseObject(Source & source) { #define EXPECT(raw, kind) \ do { \ const auto s = readString(source); \ - if (s != (raw)) { \ + if (s != (raw)) { \ throw badArchive("expected " kind " tag"); \ } \ co_yield MetadataString{s}; \ @@ -214,7 +214,7 @@ static Generator parseObject(Source & source, const Path & path) } }; auto left = size; - co_yield File{path, executable, size, makeReader(source, left)}; + co_yield File{executable, size, makeReader(source, left)}; // we could drain the remainder of the file, but coroutines being interruptible // at any time makes this difficult. for files this is not that hard, but being // consistent with directories is more important than handling the simple case. @@ -222,11 +222,18 @@ static Generator parseObject(Source & source, const Path & path) readPadding(size, source); co_yield MetadataRaw{SerializingTransform::padding(size)}; } else { - throw badArchive("file without contents found: " + path); + throw badArchive("file without contents found"); } } else if (t == "directory") { - auto makeReader = [](Source & source, const Path & path, bool & completed - ) -> Generator { + struct Map { + auto operator()(std::pair e) const { return e; } + std::pair operator()(Entry e) const { + static std::string empty; + return {empty, std::move(e)}; + } + }; + auto makeReader = [](Source & source, bool & completed + ) -> Generator, Map> { std::map names; std::string prevName; @@ -276,12 +283,15 @@ static Generator parseObject(Source & source, const Path & path) } EXPECT("node", "node"); - co_yield parseObject(source, path + "/" + name); + auto inner = parseObject(source); + while (auto i = inner.next()) { + co_yield std::pair(std::cref(name), std::move(*i)); + } EXPECT(")", "close"); } }; bool completed = false; - co_yield Directory{path, makeReader(source, path, completed)}; + co_yield Directory{makeReader(source, completed)}; // directories may nest, so to drain a directory properly we'd have to add a Finally // argument to the generator to ensure that the draining code is always run. this is // usually not necessary, hard to follow, and rather error-prone on top of all that. @@ -292,7 +302,7 @@ static Generator parseObject(Source & source, const Path & path) EXPECT("target", "target"); std::string target = readString(source); co_yield MetadataString{target}; - co_yield Symlink{path, target}; + co_yield Symlink{target}; } else { throw badArchive("unknown file type " + t); } @@ -314,55 +324,54 @@ Generator parse(Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - co_yield parseObject(source, ""); + co_yield parseObject(source); } } - -static WireFormatGenerator restore(NARParseVisitor & sink, Generator nar) +static WireFormatGenerator restore(NARParseVisitor & sink, nar::Entry entry, const Path & path) { - while (auto entry = nar.next()) { - co_yield std::visit( - overloaded{ - [](nar::MetadataString m) -> WireFormatGenerator { - co_yield m.data; - }, - [](nar::MetadataRaw r) -> WireFormatGenerator { - co_yield r.raw; - }, - [&](nar::File f) { - return [](auto f, auto & sink) -> WireFormatGenerator { - auto handle = sink.createRegularFile(f.path, f.size, f.executable); - - while (auto block = f.contents.next()) { - handle->receiveContents(std::string_view{block->data(), block->size()}); - co_yield *block; - } - handle->close(); - }(std::move(f), sink); - }, - [&](nar::Symlink sl) { - return [](auto sl, auto & sink) -> WireFormatGenerator { - sink.createSymlink(sl.path, sl.target); - co_return; - }(std::move(sl), sink); - }, - [&](nar::Directory d) { - return [](auto d, auto & sink) -> WireFormatGenerator { - auto dir = sink.createDirectory(d.path); - co_yield restore(*dir, std::move(d.contents)); - }(std::move(d), sink); - }, + return std::visit( + overloaded{ + [](nar::MetadataString m) -> WireFormatGenerator { + co_yield m.data; }, - std::move(*entry) - ); - } + [](nar::MetadataRaw r) -> WireFormatGenerator { + co_yield r.raw; + }, + [&](nar::File f) { + auto handle = sink.createRegularFile(path, f.size, f.executable); + return [](auto handle, auto f) -> WireFormatGenerator { + while (auto block = f.contents.next()) { + handle->receiveContents(std::string_view{block->data(), block->size()}); + co_yield *block; + } + handle->close(); + }(std::move(handle), std::move(f)); + }, + [&](nar::Symlink sl) { + sink.createSymlink(path, sl.target); + return []() -> WireFormatGenerator { co_return; }(); + }, + [&](nar::Directory d) { + auto dir = sink.createDirectory(path); + return [](auto path, auto dir, auto d) -> WireFormatGenerator { + while (auto entry = d.contents.next()) { + co_yield restore(*dir, std::move(entry->second), path + "/" + entry->first); + } + }(path, std::move(dir), std::move(d)); + }, + }, + std::move(entry) + ); } WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source) { - return restore(sink, nar::parse(source)); + auto nar = nar::parse(source); + while (auto entry = nar.next()) { + co_yield restore(sink, std::move(*entry), ""); + } } void parseDump(NARParseVisitor & sink, Source & source) diff --git a/lix/libutil/archive.hh b/lix/libutil/archive.hh index 6d9f841c1..331241065 100644 --- a/lix/libutil/archive.hh +++ b/lix/libutil/archive.hh @@ -138,7 +138,6 @@ struct MetadataRaw struct File { - const Path & path; bool executable; uint64_t size; Generator contents; @@ -146,14 +145,12 @@ struct File struct Symlink { - const Path & path; const Path & target; }; struct Directory { - const Path & path; - Generator contents; + Generator> contents; }; Generator parse(Source & source); diff --git a/tests/unit/libutil/archive.cc b/tests/unit/libutil/archive.cc index 16496b93a..75896b95d 100644 --- a/tests/unit/libutil/archive.cc +++ b/tests/unit/libutil/archive.cc @@ -54,7 +54,7 @@ Fragment lparen = metaString("("); Fragment rparen = metaString(")"); Fragment type = metaString("type"); -Fragment make_file(bool executable, Path path, std::string contents) +Fragment make_file(bool executable, std::string contents) { assert(contents.size() <= 255); return concat({ @@ -66,11 +66,10 @@ Fragment make_file(bool executable, Path path, std::string contents) metaString("contents"), {char(contents.size()) + "\0\0\0\0\0\0\0"s + contents + "\0\0\0\0\0\0\0\0"s.substr(0, (8 - contents.size() % 8) % 8), - [executable, path, contents] { - return [](auto executable, auto path, auto contents) -> Entries { + [executable, contents] { + return [](auto executable, auto contents) -> Entries { co_yield metaRaw(char(contents.size()) + "\0\0\0\0\0\0\0"s)(); co_yield File{ - path, executable, contents.size(), [](auto contents) -> Generator { @@ -78,13 +77,13 @@ Fragment make_file(bool executable, Path path, std::string contents) }(contents) }; co_yield metaRaw("\0\0\0\0\0\0\0\0"sv.substr(0, (8 - contents.size() % 8) % 8))(); - }(executable, path, contents); + }(executable, contents); }}, rparen, }); } -Fragment make_symlink(Path path, std::string linkTarget) +Fragment make_symlink(std::string linkTarget) { assert(linkTarget.size() <= 255); return concat({ @@ -95,45 +94,55 @@ Fragment make_symlink(Path path, std::string linkTarget) metaString(linkTarget), { "", - [path, linkTarget] { - return [](auto link) -> Entries { co_yield link; }(Symlink{path, linkTarget}); + [linkTarget] { + return [](auto link) -> Entries { co_yield link; }(Symlink{linkTarget}); }, }, rparen, }); } -Fragment make_directory(Path path, std::vector> entries) +Fragment make_directory(std::vector> entries) { - std::vector parts; + std::string raw; + std::vector>> inodes; + + auto append = [&](std::string_view name, Fragment f) { + raw += f.first; + inodes.emplace_back(name, f.second); + }; + for (auto & [dentryName, dentry] : entries) { assert(dentryName.size() <= 255); - parts.push_back(metaString("entry")); - parts.push_back(lparen); - parts.push_back(metaString("name")); - parts.emplace_back(metaString(dentryName)); - parts.push_back(metaString("node")); - parts.push_back(dentry); - parts.push_back(rparen); + append("", metaString("entry")); + append("", lparen); + append("", metaString("name")); + append("", metaString(dentryName)); + append("", metaString("node")); + append(dentryName, dentry); + append("", rparen); } - auto inner = concat(parts); - return concat({ lparen, type, metaString("directory"), { - inner.first + rparen.first, - [path, inner] { + raw + rparen.first, + [inodes] { return ([](auto dir) -> Entries { co_yield std::move(dir); })(Directory{ - path, - [](auto inner) -> Entries { - co_yield std::move(inner); + [](auto inodes) -> Generator> { + for (auto & [name, dentry] : inodes) { + auto subs = dentry(); + while (auto si = subs.next()) { + co_yield std::pair(std::cref(name), std::move(*si)); + } + } // this should be a separate item, but the parser emits it // from within the directory. but as long as it's there... - co_yield rparen.second(); - }(inner.second()), + std::string empty; + co_yield std::pair{empty, *rparen.second().next()}; + }(inodes), }); }, }, @@ -152,7 +161,6 @@ void assert_eq(const MetadataRaw & a, const MetadataRaw & b) } void assert_eq(File & a, File & b) { - ASSERT_EQ(a.path, b.path); ASSERT_EQ(a.executable, b.executable); ASSERT_EQ(a.size, b.size); auto acontents = GeneratorSource(std::move(a.contents)).drain(); @@ -161,12 +169,10 @@ void assert_eq(File & a, File & b) } void assert_eq(const Symlink & a, const Symlink & b) { - ASSERT_EQ(a.path, b.path); ASSERT_EQ(a.target, b.target); } void assert_eq(Directory & a, Directory & b) { - ASSERT_EQ(a.path, b.path); while (true) { auto ae = a.contents.next(); auto be = b.contents.next(); @@ -174,7 +180,8 @@ void assert_eq(Directory & a, Directory & b) if (!ae.has_value()) { break; } - assert_eq(*ae, *be); + ASSERT_EQ(ae->first, be->first); + assert_eq(ae->second, be->second); } } @@ -217,44 +224,44 @@ INSTANTIATE_TEST_SUITE_P( , NarTest, testing::Values( - concat({header, make_file(false, "", "")}), - concat({header, make_file(false, "", "short")}), - concat({header, make_file(false, "", "block000")}), - concat({header, make_file(false, "", "block0001")}), - concat({header, make_file(true, "", "")}), - concat({header, make_file(true, "", "short")}), - concat({header, make_file(true, "", "block000")}), - concat({header, make_file(true, "", "block0001")}), - concat({header, make_symlink("", "")}), - concat({header, make_symlink("", "short")}), - concat({header, make_symlink("", "block000")}), - concat({header, make_symlink("", "block0001")}), + concat({header, make_file(false, "")}), + concat({header, make_file(false, "short")}), + concat({header, make_file(false, "block000")}), + concat({header, make_file(false, "block0001")}), + concat({header, make_file(true, "")}), + concat({header, make_file(true, "short")}), + concat({header, make_file(true, "block000")}), + concat({header, make_file(true, "block0001")}), + concat({header, make_symlink("")}), + concat({header, make_symlink("short")}), + concat({header, make_symlink("block000")}), + concat({header, make_symlink("block0001")}), - concat({header, make_directory("", {{"a", make_file(false, "/a", "")}})}), - concat({header, make_directory("", {{"a", make_file(false, "/a", "short")}})}), - concat({header, make_directory("", {{"a", make_file(false, "/a", "block000")}})}), - concat({header, make_directory("", {{"a", make_file(false, "/a", "block0001")}})}), - concat({header, make_directory("", {{"a", make_file(true, "/a", "")}})}), - concat({header, make_directory("", {{"a", make_file(true, "/a", "short")}})}), - concat({header, make_directory("", {{"a", make_file(true, "/a", "block000")}})}), - concat({header, make_directory("", {{"a", make_file(true, "/a", "block0001")}})}), - concat({header, make_directory("", {{"a", make_symlink("/a", "")}})}), - concat({header, make_directory("", {{"a", make_symlink("/a", "short")}})}), - concat({header, make_directory("", {{"a", make_symlink("/a", "block000")}})}), - concat({header, make_directory("", {{"a", make_symlink("/a", "block0001")}})}), + concat({header, make_directory({{"a", make_file(false, "")}})}), + concat({header, make_directory({{"a", make_file(false, "short")}})}), + concat({header, make_directory({{"a", make_file(false, "block000")}})}), + concat({header, make_directory({{"a", make_file(false, "block0001")}})}), + concat({header, make_directory({{"a", make_file(true, "")}})}), + concat({header, make_directory({{"a", make_file(true, "short")}})}), + concat({header, make_directory({{"a", make_file(true, "block000")}})}), + concat({header, make_directory({{"a", make_file(true, "block0001")}})}), + concat({header, make_directory({{"a", make_symlink("")}})}), + concat({header, make_directory({{"a", make_symlink("short")}})}), + concat({header, make_directory({{"a", make_symlink("block000")}})}), + concat({header, make_directory({{"a", make_symlink("block0001")}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "short")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "block000")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "block0001")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "short")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "block000")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "block0001")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "short")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "block000")}})}})}), - concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "block0001")}})}})}) + concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "short")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "block000")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "block0001")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "short")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "block000")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "block0001")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_symlink("")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_symlink("short")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_symlink("block000")}})}})}), + concat({header, make_directory({{"d", make_directory({{"a", make_symlink("block0001")}})}})}) ) ); }