From 66d515a3a2d01e43b9560374996080d694a0f4d7 Mon Sep 17 00:00:00 2001
From: eldritch horrors <pennae@lix.systems>
Date: Mon, 3 Mar 2025 20:48:59 +0100
Subject: [PATCH] libutil: add an async nar indexer

Change-Id: I4c22364c94e2bea1f18b835d9514c4c65a530c80
---
 lix/libutil/archive.cc        | 57 +++++++++++++++++++++++++++--------
 lix/libutil/archive.hh        |  1 +
 tests/unit/libutil/archive.cc | 20 ++++++++++++
 3 files changed, 65 insertions(+), 13 deletions(-)

diff --git a/lix/libutil/archive.cc b/lix/libutil/archive.cc
index 581a241bc..2cc3a706a 100644
--- a/lix/libutil/archive.cc
+++ b/lix/libutil/archive.cc
@@ -752,28 +752,18 @@ Generator<Entry> parse(Source & source)
 namespace nar_index {
 
 namespace {
-struct NarSource : Source
+struct NarPositioner
 {
-    Source & source;
     uint64_t pos = 0;
-
-    NarSource(Source & source) : source(source) {}
-
-    size_t read(char * data, size_t len) override
-    {
-        auto n = source.read(data, len);
-        pos += n;
-        return n;
-    }
 };
 
 struct Indexer : NARParseVisitor
 {
-    NarSource & source;
+    NarPositioner & source;
     Directory & parent;
 
 public:
-    Indexer(NarSource & source, Directory & parent) : source(source), parent(parent) {}
+    Indexer(NarPositioner & source, Directory & parent) : source(source), parent(parent) {}
 
     box_ptr<NARParseVisitor> createDirectory(const std::string & name) override
     {
@@ -803,6 +793,20 @@ public:
 
 Entry create(Source & source)
 {
+    struct NarSource : Source, NarPositioner
+    {
+        Source & source;
+
+        NarSource(Source & source) : source(source) {}
+
+        size_t read(char * data, size_t len) override
+        {
+            auto n = source.read(data, len);
+            pos += n;
+            return n;
+        }
+    };
+
     Directory root;
     NarSource wrapper{source};
     Indexer index{wrapper, root};
@@ -810,6 +814,33 @@ Entry create(Source & source)
     return root.contents.at("");
 }
 
+kj::Promise<Result<Entry>> create(AsyncInputStream & source)
+try {
+    struct NarSource : AsyncInputStream, NarPositioner
+    {
+        AsyncInputStream & source;
+
+        NarSource(AsyncInputStream & source) : source(source) {}
+
+        kj::Promise<Result<size_t>> read(void * data, size_t len) override
+        try {
+            auto n = TRY_AWAIT(source.read(data, len));
+            pos += n;
+            co_return n;
+        } catch (...) {
+            co_return result::current_exception();
+        }
+    };
+
+    Directory root;
+    NarSource wrapper{source};
+    Indexer index{wrapper, root};
+    TRY_AWAIT(parseDump(index, wrapper));
+    co_return root.contents.at("");
+} catch (...) {
+    co_return result::current_exception();
+}
+
 }
 
 static void restore(NARParseVisitor & sink, nar::Entry entry, const Path & path)
diff --git a/lix/libutil/archive.hh b/lix/libutil/archive.hh
index 2f11361f2..3136becfc 100644
--- a/lix/libutil/archive.hh
+++ b/lix/libutil/archive.hh
@@ -200,6 +200,7 @@ struct Directory
 };
 
 Entry create(Source & source);
+kj::Promise<Result<Entry>> create(AsyncInputStream & source);
 
 }
 
diff --git a/tests/unit/libutil/archive.cc b/tests/unit/libutil/archive.cc
index e0545d325..605448a91 100644
--- a/tests/unit/libutil/archive.cc
+++ b/tests/unit/libutil/archive.cc
@@ -364,6 +364,26 @@ TEST_P(NarTest, index)
     }
 }
 
+TEST_P(NarTest, indexAsync)
+{
+    auto & [raw, entriesF] = GetParam();
+    AsyncStringInputStream source(raw);
+
+    kj::EventLoop el;
+    kj::WaitScope ws{el};
+    auto entries = entriesF();
+    auto indexed = fromIndex(raw, nar_index::create(source).wait(ws).value());
+    while (true) {
+        auto e = entries.next();
+        auto p = indexed.next();
+        ASSERT_EQ(e.has_value(), p.has_value());
+        if (!e) {
+            break;
+        }
+        assert_eq(*e, *p);
+    }
+}
+
 INSTANTIATE_TEST_SUITE_P(
     ,
     NarTest,