libutil: attach nar member names to dentries, not inodes
this is where they should've been from the start, but during the first rewrite it made little sense to move them. we have bigger plans today, so we'll finally clean that up too. note the `Map` transform type that is needed to make the current macros work. it shall be only temporary. Change-Id: I928d197dbfe27b68cf8634d149c3259e86fbf123
This commit is contained in:
eldritch horrors
+57
-48
@@ -167,12 +167,12 @@ struct CaseInsensitiveCompare
|
||||
|
||||
namespace nar {
|
||||
|
||||
static Generator<Entry> parseObject(Source & source, const Path & path)
|
||||
static Generator<Entry> parseObject(Source & source)
|
||||
{
|
||||
#define EXPECT(raw, kind) \
|
||||
do { \
|
||||
const auto s = readString(source); \
|
||||
if (s != (raw)) { \
|
||||
if (s != (raw)) { \
|
||||
throw badArchive("expected " kind " tag"); \
|
||||
} \
|
||||
co_yield MetadataString{s}; \
|
||||
@@ -214,7 +214,7 @@ static Generator<Entry> parseObject(Source & source, const Path & path)
|
||||
}
|
||||
};
|
||||
auto left = size;
|
||||
co_yield File{path, executable, size, makeReader(source, left)};
|
||||
co_yield File{executable, size, makeReader(source, left)};
|
||||
// we could drain the remainder of the file, but coroutines being interruptible
|
||||
// at any time makes this difficult. for files this is not that hard, but being
|
||||
// consistent with directories is more important than handling the simple case.
|
||||
@@ -222,11 +222,18 @@ static Generator<Entry> parseObject(Source & source, const Path & path)
|
||||
readPadding(size, source);
|
||||
co_yield MetadataRaw{SerializingTransform::padding(size)};
|
||||
} else {
|
||||
throw badArchive("file without contents found: " + path);
|
||||
throw badArchive("file without contents found");
|
||||
}
|
||||
} else if (t == "directory") {
|
||||
auto makeReader = [](Source & source, const Path & path, bool & completed
|
||||
) -> Generator<Entry> {
|
||||
struct Map {
|
||||
auto operator()(std::pair<const std::string &, Entry> e) const { return e; }
|
||||
std::pair<const std::string &, Entry> operator()(Entry e) const {
|
||||
static std::string empty;
|
||||
return {empty, std::move(e)};
|
||||
}
|
||||
};
|
||||
auto makeReader = [](Source & source, bool & completed
|
||||
) -> Generator<std::pair<const std::string &, Entry>, Map> {
|
||||
std::map<Path, int, CaseInsensitiveCompare> names;
|
||||
std::string prevName;
|
||||
|
||||
@@ -276,12 +283,15 @@ static Generator<Entry> parseObject(Source & source, const Path & path)
|
||||
}
|
||||
|
||||
EXPECT("node", "node");
|
||||
co_yield parseObject(source, path + "/" + name);
|
||||
auto inner = parseObject(source);
|
||||
while (auto i = inner.next()) {
|
||||
co_yield std::pair(std::cref(name), std::move(*i));
|
||||
}
|
||||
EXPECT(")", "close");
|
||||
}
|
||||
};
|
||||
bool completed = false;
|
||||
co_yield Directory{path, makeReader(source, path, completed)};
|
||||
co_yield Directory{makeReader(source, completed)};
|
||||
// directories may nest, so to drain a directory properly we'd have to add a Finally
|
||||
// argument to the generator to ensure that the draining code is always run. this is
|
||||
// usually not necessary, hard to follow, and rather error-prone on top of all that.
|
||||
@@ -292,7 +302,7 @@ static Generator<Entry> parseObject(Source & source, const Path & path)
|
||||
EXPECT("target", "target");
|
||||
std::string target = readString(source);
|
||||
co_yield MetadataString{target};
|
||||
co_yield Symlink{path, target};
|
||||
co_yield Symlink{target};
|
||||
} else {
|
||||
throw badArchive("unknown file type " + t);
|
||||
}
|
||||
@@ -314,55 +324,54 @@ Generator<Entry> parse(Source & source)
|
||||
}
|
||||
if (version != narVersionMagic1)
|
||||
throw badArchive("input doesn't look like a Nix archive");
|
||||
co_yield parseObject(source, "");
|
||||
co_yield parseObject(source);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
static WireFormatGenerator restore(NARParseVisitor & sink, Generator<nar::Entry> nar)
|
||||
static WireFormatGenerator restore(NARParseVisitor & sink, nar::Entry entry, const Path & path)
|
||||
{
|
||||
while (auto entry = nar.next()) {
|
||||
co_yield std::visit(
|
||||
overloaded{
|
||||
[](nar::MetadataString m) -> WireFormatGenerator {
|
||||
co_yield m.data;
|
||||
},
|
||||
[](nar::MetadataRaw r) -> WireFormatGenerator {
|
||||
co_yield r.raw;
|
||||
},
|
||||
[&](nar::File f) {
|
||||
return [](auto f, auto & sink) -> WireFormatGenerator {
|
||||
auto handle = sink.createRegularFile(f.path, f.size, f.executable);
|
||||
|
||||
while (auto block = f.contents.next()) {
|
||||
handle->receiveContents(std::string_view{block->data(), block->size()});
|
||||
co_yield *block;
|
||||
}
|
||||
handle->close();
|
||||
}(std::move(f), sink);
|
||||
},
|
||||
[&](nar::Symlink sl) {
|
||||
return [](auto sl, auto & sink) -> WireFormatGenerator {
|
||||
sink.createSymlink(sl.path, sl.target);
|
||||
co_return;
|
||||
}(std::move(sl), sink);
|
||||
},
|
||||
[&](nar::Directory d) {
|
||||
return [](auto d, auto & sink) -> WireFormatGenerator {
|
||||
auto dir = sink.createDirectory(d.path);
|
||||
co_yield restore(*dir, std::move(d.contents));
|
||||
}(std::move(d), sink);
|
||||
},
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[](nar::MetadataString m) -> WireFormatGenerator {
|
||||
co_yield m.data;
|
||||
},
|
||||
std::move(*entry)
|
||||
);
|
||||
}
|
||||
[](nar::MetadataRaw r) -> WireFormatGenerator {
|
||||
co_yield r.raw;
|
||||
},
|
||||
[&](nar::File f) {
|
||||
auto handle = sink.createRegularFile(path, f.size, f.executable);
|
||||
return [](auto handle, auto f) -> WireFormatGenerator {
|
||||
while (auto block = f.contents.next()) {
|
||||
handle->receiveContents(std::string_view{block->data(), block->size()});
|
||||
co_yield *block;
|
||||
}
|
||||
handle->close();
|
||||
}(std::move(handle), std::move(f));
|
||||
},
|
||||
[&](nar::Symlink sl) {
|
||||
sink.createSymlink(path, sl.target);
|
||||
return []() -> WireFormatGenerator { co_return; }();
|
||||
},
|
||||
[&](nar::Directory d) {
|
||||
auto dir = sink.createDirectory(path);
|
||||
return [](auto path, auto dir, auto d) -> WireFormatGenerator {
|
||||
while (auto entry = d.contents.next()) {
|
||||
co_yield restore(*dir, std::move(entry->second), path + "/" + entry->first);
|
||||
}
|
||||
}(path, std::move(dir), std::move(d));
|
||||
},
|
||||
},
|
||||
std::move(entry)
|
||||
);
|
||||
}
|
||||
|
||||
WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source)
|
||||
{
|
||||
return restore(sink, nar::parse(source));
|
||||
auto nar = nar::parse(source);
|
||||
while (auto entry = nar.next()) {
|
||||
co_yield restore(sink, std::move(*entry), "");
|
||||
}
|
||||
}
|
||||
|
||||
void parseDump(NARParseVisitor & sink, Source & source)
|
||||
|
||||
@@ -138,7 +138,6 @@ struct MetadataRaw
|
||||
|
||||
struct File
|
||||
{
|
||||
const Path & path;
|
||||
bool executable;
|
||||
uint64_t size;
|
||||
Generator<Bytes> contents;
|
||||
@@ -146,14 +145,12 @@ struct File
|
||||
|
||||
struct Symlink
|
||||
{
|
||||
const Path & path;
|
||||
const Path & target;
|
||||
};
|
||||
|
||||
struct Directory
|
||||
{
|
||||
const Path & path;
|
||||
Generator<Entry> contents;
|
||||
Generator<std::pair<const std::string &, Entry>> contents;
|
||||
};
|
||||
|
||||
Generator<Entry> parse(Source & source);
|
||||
|
||||
@@ -54,7 +54,7 @@ Fragment lparen = metaString("(");
|
||||
Fragment rparen = metaString(")");
|
||||
Fragment type = metaString("type");
|
||||
|
||||
Fragment make_file(bool executable, Path path, std::string contents)
|
||||
Fragment make_file(bool executable, std::string contents)
|
||||
{
|
||||
assert(contents.size() <= 255);
|
||||
return concat({
|
||||
@@ -66,11 +66,10 @@ Fragment make_file(bool executable, Path path, std::string contents)
|
||||
metaString("contents"),
|
||||
{char(contents.size()) + "\0\0\0\0\0\0\0"s + contents
|
||||
+ "\0\0\0\0\0\0\0\0"s.substr(0, (8 - contents.size() % 8) % 8),
|
||||
[executable, path, contents] {
|
||||
return [](auto executable, auto path, auto contents) -> Entries {
|
||||
[executable, contents] {
|
||||
return [](auto executable, auto contents) -> Entries {
|
||||
co_yield metaRaw(char(contents.size()) + "\0\0\0\0\0\0\0"s)();
|
||||
co_yield File{
|
||||
path,
|
||||
executable,
|
||||
contents.size(),
|
||||
[](auto contents) -> Generator<Bytes> {
|
||||
@@ -78,13 +77,13 @@ Fragment make_file(bool executable, Path path, std::string contents)
|
||||
}(contents)
|
||||
};
|
||||
co_yield metaRaw("\0\0\0\0\0\0\0\0"sv.substr(0, (8 - contents.size() % 8) % 8))();
|
||||
}(executable, path, contents);
|
||||
}(executable, contents);
|
||||
}},
|
||||
rparen,
|
||||
});
|
||||
}
|
||||
|
||||
Fragment make_symlink(Path path, std::string linkTarget)
|
||||
Fragment make_symlink(std::string linkTarget)
|
||||
{
|
||||
assert(linkTarget.size() <= 255);
|
||||
return concat({
|
||||
@@ -95,45 +94,55 @@ Fragment make_symlink(Path path, std::string linkTarget)
|
||||
metaString(linkTarget),
|
||||
{
|
||||
"",
|
||||
[path, linkTarget] {
|
||||
return [](auto link) -> Entries { co_yield link; }(Symlink{path, linkTarget});
|
||||
[linkTarget] {
|
||||
return [](auto link) -> Entries { co_yield link; }(Symlink{linkTarget});
|
||||
},
|
||||
},
|
||||
rparen,
|
||||
});
|
||||
}
|
||||
|
||||
Fragment make_directory(Path path, std::vector<std::pair<std::string, Fragment>> entries)
|
||||
Fragment make_directory(std::vector<std::pair<std::string, Fragment>> entries)
|
||||
{
|
||||
std::vector<Fragment> parts;
|
||||
std::string raw;
|
||||
std::vector<std::pair<std::string, std::function<Entries()>>> inodes;
|
||||
|
||||
auto append = [&](std::string_view name, Fragment f) {
|
||||
raw += f.first;
|
||||
inodes.emplace_back(name, f.second);
|
||||
};
|
||||
|
||||
for (auto & [dentryName, dentry] : entries) {
|
||||
assert(dentryName.size() <= 255);
|
||||
parts.push_back(metaString("entry"));
|
||||
parts.push_back(lparen);
|
||||
parts.push_back(metaString("name"));
|
||||
parts.emplace_back(metaString(dentryName));
|
||||
parts.push_back(metaString("node"));
|
||||
parts.push_back(dentry);
|
||||
parts.push_back(rparen);
|
||||
append("", metaString("entry"));
|
||||
append("", lparen);
|
||||
append("", metaString("name"));
|
||||
append("", metaString(dentryName));
|
||||
append("", metaString("node"));
|
||||
append(dentryName, dentry);
|
||||
append("", rparen);
|
||||
}
|
||||
|
||||
auto inner = concat(parts);
|
||||
|
||||
return concat({
|
||||
lparen,
|
||||
type,
|
||||
metaString("directory"),
|
||||
{
|
||||
inner.first + rparen.first,
|
||||
[path, inner] {
|
||||
raw + rparen.first,
|
||||
[inodes] {
|
||||
return ([](auto dir) -> Entries { co_yield std::move(dir); })(Directory{
|
||||
path,
|
||||
[](auto inner) -> Entries {
|
||||
co_yield std::move(inner);
|
||||
[](auto inodes) -> Generator<std::pair<const std::string &, Entry>> {
|
||||
for (auto & [name, dentry] : inodes) {
|
||||
auto subs = dentry();
|
||||
while (auto si = subs.next()) {
|
||||
co_yield std::pair(std::cref(name), std::move(*si));
|
||||
}
|
||||
}
|
||||
// this should be a separate item, but the parser emits it
|
||||
// from within the directory. but as long as it's there...
|
||||
co_yield rparen.second();
|
||||
}(inner.second()),
|
||||
std::string empty;
|
||||
co_yield std::pair{empty, *rparen.second().next()};
|
||||
}(inodes),
|
||||
});
|
||||
},
|
||||
},
|
||||
@@ -152,7 +161,6 @@ void assert_eq(const MetadataRaw & a, const MetadataRaw & b)
|
||||
}
|
||||
void assert_eq(File & a, File & b)
|
||||
{
|
||||
ASSERT_EQ(a.path, b.path);
|
||||
ASSERT_EQ(a.executable, b.executable);
|
||||
ASSERT_EQ(a.size, b.size);
|
||||
auto acontents = GeneratorSource(std::move(a.contents)).drain();
|
||||
@@ -161,12 +169,10 @@ void assert_eq(File & a, File & b)
|
||||
}
|
||||
void assert_eq(const Symlink & a, const Symlink & b)
|
||||
{
|
||||
ASSERT_EQ(a.path, b.path);
|
||||
ASSERT_EQ(a.target, b.target);
|
||||
}
|
||||
void assert_eq(Directory & a, Directory & b)
|
||||
{
|
||||
ASSERT_EQ(a.path, b.path);
|
||||
while (true) {
|
||||
auto ae = a.contents.next();
|
||||
auto be = b.contents.next();
|
||||
@@ -174,7 +180,8 @@ void assert_eq(Directory & a, Directory & b)
|
||||
if (!ae.has_value()) {
|
||||
break;
|
||||
}
|
||||
assert_eq(*ae, *be);
|
||||
ASSERT_EQ(ae->first, be->first);
|
||||
assert_eq(ae->second, be->second);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,44 +224,44 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
,
|
||||
NarTest,
|
||||
testing::Values(
|
||||
concat({header, make_file(false, "", "")}),
|
||||
concat({header, make_file(false, "", "short")}),
|
||||
concat({header, make_file(false, "", "block000")}),
|
||||
concat({header, make_file(false, "", "block0001")}),
|
||||
concat({header, make_file(true, "", "")}),
|
||||
concat({header, make_file(true, "", "short")}),
|
||||
concat({header, make_file(true, "", "block000")}),
|
||||
concat({header, make_file(true, "", "block0001")}),
|
||||
concat({header, make_symlink("", "")}),
|
||||
concat({header, make_symlink("", "short")}),
|
||||
concat({header, make_symlink("", "block000")}),
|
||||
concat({header, make_symlink("", "block0001")}),
|
||||
concat({header, make_file(false, "")}),
|
||||
concat({header, make_file(false, "short")}),
|
||||
concat({header, make_file(false, "block000")}),
|
||||
concat({header, make_file(false, "block0001")}),
|
||||
concat({header, make_file(true, "")}),
|
||||
concat({header, make_file(true, "short")}),
|
||||
concat({header, make_file(true, "block000")}),
|
||||
concat({header, make_file(true, "block0001")}),
|
||||
concat({header, make_symlink("")}),
|
||||
concat({header, make_symlink("short")}),
|
||||
concat({header, make_symlink("block000")}),
|
||||
concat({header, make_symlink("block0001")}),
|
||||
|
||||
concat({header, make_directory("", {{"a", make_file(false, "/a", "")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(false, "/a", "short")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(false, "/a", "block000")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(false, "/a", "block0001")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(true, "/a", "")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(true, "/a", "short")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(true, "/a", "block000")}})}),
|
||||
concat({header, make_directory("", {{"a", make_file(true, "/a", "block0001")}})}),
|
||||
concat({header, make_directory("", {{"a", make_symlink("/a", "")}})}),
|
||||
concat({header, make_directory("", {{"a", make_symlink("/a", "short")}})}),
|
||||
concat({header, make_directory("", {{"a", make_symlink("/a", "block000")}})}),
|
||||
concat({header, make_directory("", {{"a", make_symlink("/a", "block0001")}})}),
|
||||
concat({header, make_directory({{"a", make_file(false, "")}})}),
|
||||
concat({header, make_directory({{"a", make_file(false, "short")}})}),
|
||||
concat({header, make_directory({{"a", make_file(false, "block000")}})}),
|
||||
concat({header, make_directory({{"a", make_file(false, "block0001")}})}),
|
||||
concat({header, make_directory({{"a", make_file(true, "")}})}),
|
||||
concat({header, make_directory({{"a", make_file(true, "short")}})}),
|
||||
concat({header, make_directory({{"a", make_file(true, "block000")}})}),
|
||||
concat({header, make_directory({{"a", make_file(true, "block0001")}})}),
|
||||
concat({header, make_directory({{"a", make_symlink("")}})}),
|
||||
concat({header, make_directory({{"a", make_symlink("short")}})}),
|
||||
concat({header, make_directory({{"a", make_symlink("block000")}})}),
|
||||
concat({header, make_directory({{"a", make_symlink("block0001")}})}),
|
||||
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "short")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "block000")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(false, "/d/a", "block0001")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "short")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "block000")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_file(true, "/d/a", "block0001")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "short")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "block000")}})}})}),
|
||||
concat({header, make_directory("", {{"d", make_directory("/d", {{"a", make_symlink("/d/a", "block0001")}})}})})
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "short")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "block000")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(false, "block0001")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "short")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "block000")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_file(true, "block0001")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_symlink("")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_symlink("short")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_symlink("block000")}})}})}),
|
||||
concat({header, make_directory({{"d", make_directory({{"a", make_symlink("block0001")}})}})})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user