From 23675210083fcf05a2ed5770a0aea7684017ade5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 3 Mar 2025 20:48:59 +0100 Subject: [PATCH] libutil: add an async nar parser sadly this is a visitor-only interface; async generators are not yet a thing and preliminary benchmarks say that overhead would be too large. Change-Id: I0460d18eba94441cf3101d46bfffdb69cdda81d1 --- lix/libutil/archive.cc | 94 ++++++++++++++++++++++++++++++++- lix/libutil/archive.hh | 1 + tests/unit/libutil/archive.cc | 98 +++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) diff --git a/lix/libutil/archive.cc b/lix/libutil/archive.cc index ef38af6d7..581a241bc 100644 --- a/lix/libutil/archive.cc +++ b/lix/libutil/archive.cc @@ -15,12 +15,13 @@ #include "lix/libutil/archive.hh" #include "lix/libutil/async-io.hh" #include "lix/libutil/box_ptr.hh" +#include "lix/libutil/config.hh" #include "lix/libutil/file-system.hh" #include "lix/libutil/finally.hh" #include "lix/libutil/generator.hh" -#include "lix/libutil/serialise.hh" -#include "lix/libutil/config.hh" #include "lix/libutil/logging.hh" +#include "lix/libutil/result.hh" +#include "lix/libutil/serialise.hh" #include "lix/libutil/signals.hh" namespace nix { @@ -658,6 +659,86 @@ struct AsyncCopier : AsyncInputStream } } }; + +// sadly async parsers can't be written to produce a tree of generators +// the way sync parsers can. once we have async generators that may not +// be as hard, but async generators in kj might have too much overhead. +struct AsyncParser +{ + AsyncInputStream & source; + std::vector buffer; + + kj::Promise> parse(NARParseVisitor & target) + try { + Parser parser{buffer}; + auto stream = parser.parseRoot(); + TRY_AWAIT(parse(stream, target, "")); + co_return result::success(); + } catch (...) { + co_return result::current_exception(); + } + + kj::Promise> read(char * buffer, size_t n) + try { + while (n > 0) { + auto got = TRY_AWAIT(source.read(buffer, n)); + if (got == 0) { + throw badArchive("unexpected end of nar encountered"); + } + buffer += got; + n -= got; + } + co_return result::success(); + } catch (...) { + co_return result::current_exception(); + } + + kj::Promise> feed(size_t n) + { + auto end = buffer.size(); + buffer.resize(end + n); + return read(buffer.data() + end, n); + } + + kj::Promise> + parse(Generator & stream, NARParseVisitor & target, const std::string & name) + try { + while (auto i = stream.next()) { + if (auto want = std::get_if(&*i)) { + TRY_AWAIT(feed(want->n)); + } else if (auto f = std::get_if(&*i)) { + auto file = target.createRegularFile(name, f->size, f->executable); + auto left = f->size; + std::array buf; + + while (left) { + auto n = size_t(std::min(buf.size(), left)); + TRY_AWAIT(read(buf.data(), n)); + file->receiveContents({buf.data(), n}); + left -= n; + } + } else if (auto sl = std::get_if(&*i)) { + target.createSymlink(name, sl->target); + } else if (auto d = std::get_if(&*i)) { + auto dir = target.createDirectory(name); + while (auto e = d->content.next()) { + if (auto want = std::get_if(&*e)) { + TRY_AWAIT(feed(want->n)); + } else if (auto entry = std::get_if(&*e)) { + TRY_AWAIT(parse(entry->second, *dir, entry->first)); + } else { + assert(false && "expected parser response in dir"); + } + } + } else { + assert(false && "unhandled parser response"); + } + } + co_return result::success(); + } catch (...) { + co_return result::current_exception(); + } +}; } Generator parse(Source & source) @@ -762,6 +843,15 @@ void parseDump(NARParseVisitor & sink, Source & source) } } +kj::Promise> parseDump(NARParseVisitor & sink, AsyncInputStream & source) +try { + nar::AsyncParser parser{source}; + TRY_AWAIT(parser.parse(sink)); + co_return result::success(); +} catch (...) { + co_return result::current_exception(); +} + /* * Note [NAR restoration security]: * It's *critical* that NAR restoration will never overwrite anything even if diff --git a/lix/libutil/archive.hh b/lix/libutil/archive.hh index cb72269c9..2f11361f2 100644 --- a/lix/libutil/archive.hh +++ b/lix/libutil/archive.hh @@ -204,6 +204,7 @@ Entry create(Source & source); } void parseDump(NARParseVisitor & sink, Source & source); +kj::Promise> parseDump(NARParseVisitor & sink, AsyncInputStream & source); void restorePath(const Path & path, Source & source); diff --git a/tests/unit/libutil/archive.cc b/tests/unit/libutil/archive.cc index 219cdc4a5..e0545d325 100644 --- a/tests/unit/libutil/archive.cc +++ b/tests/unit/libutil/archive.cc @@ -1,5 +1,6 @@ #include "lix/libutil/archive.hh" #include "lix/libutil/async-io.hh" +#include "lix/libutil/box_ptr.hh" #include "lix/libutil/serialise.hh" #include #include @@ -228,6 +229,103 @@ TEST_P(NarTest, parse) } } +TEST_P(NarTest, parseAsync) +{ + struct File + { + bool executable; + uint64_t size; + std::string contents; + operator nar::Entry() const + { + auto stream = [](auto contents) -> Generator { co_yield contents; }; + return nar::File{executable, size, stream(std::span{contents})}; + } + }; + struct Directory; + using Entry = std::variant; + + struct Directory : std::map + { + operator nar::Entry() const + { + return nar::Directory{ + [](const Directory & d) -> Generator> { + for (auto & [name, entry] : d) { + co_yield std::pair{std::cref(name), *toNar(entry).next()}; + } + }(*this), + }; + } + + static Entries toNar(const Entry & e) + { + co_yield std::visit([](const auto & e) -> Entries { co_yield e; }, e); + } + }; + + struct ReconstructVisitor : NARParseVisitor + { + std::map & parent; + + struct FileReader : NARParseVisitor::FileHandle + { + File & file; + + FileReader(File & file) : file(file) {} + + void receiveContents(std::string_view data) override + { + file.contents += data; + } + + void close() override {} + }; + + explicit ReconstructVisitor(std::map & parent) : parent(parent) {} + + box_ptr createDirectory(const std::string & name) override + { + auto & dir = std::get(parent.emplace(name, Directory{}).first->second); + return make_box_ptr(dir); + } + + box_ptr + createRegularFile(const std::string & name, uint64_t size, bool executable) override + { + auto & file = std::get(parent.emplace(name, File{executable, size}).first->second); + return make_box_ptr(file); + } + + void createSymlink(const std::string & name, const std::string & target) override + { + parent.emplace(name, Symlink{target}); + } + }; + + auto & [raw, entriesF] = GetParam(); + AsyncStringInputStream source(raw); + + std::map contents; + ReconstructVisitor rv{contents}; + + kj::EventLoop el; + kj::WaitScope ws{el}; + + auto entries = entriesF(); + parseDump(rv, source).wait(ws).value(); + auto parsed = Directory::toNar(contents.at("")); + while (true) { + auto e = entries.next(); + auto p = parsed.next(); + ASSERT_EQ(e.has_value(), p.has_value()); + if (!e) { + break; + } + assert_eq(*e, *p); + } +} + TEST_P(NarTest, copy) { auto & [raw, _] = GetParam();