jade wip

Change-Id: I4d5a0bebeeefc35dac0bbc065c104c70c24aae17

lix/libutil/compression.cc

@@ -16,7 +16,6 @@
 #include <archive.h>
 #include <archive_entry.h>
 #include <cerrno>
-#include <chrono>
 #include <cstdio>
 #include <cstring>
 
@@ -63,13 +62,15 @@ struct ArchiveDecompressionSource : Source
         struct archive_entry * ae;
         if (!archive) {
             archive = std::make_unique<TarArchive>(*src, true);
-            this->archive->check(archive_read_next_header(this->archive->archive, &ae),
-                "failed to read header (%s)");
-            if (archive_filter_count(this->archive->archive) < 2) {
+            this->archive->check(
+                archive_read_next_header(this->archive->archive.get(), &ae),
+                "failed to read header (%s)"
+            );
+            if (archive_filter_count(this->archive->archive.get()) < 2) {
                 throw CompressionError("input compression not recognized");
             }
         }
-        ssize_t result = archive_read_data(this->archive->archive, data, len);
+        ssize_t result = archive_read_data(this->archive->archive.get(), data, len);
         if (result > 0) return result;
         if (result == 0) {
             throw EndOfFile("reached end of compressed file");

lix/libutil/tarfile.cc

@@ -1,9 +1,11 @@
 #include <archive.h>
 #include <archive_entry.h>
 #include <kj/async.h>
+#include <map>
 
 #include "async-io.hh"
 #include "file-descriptor.hh"
+#include "lix/libstore/temporary-dir.hh"
 #include "lix/libutil/c-calls.hh"
 #include "lix/libutil/charptr-cast.hh"
 #include "lix/libutil/file-system.hh"
@@ -45,67 +47,110 @@ void TarArchive::check(int err, const std::string & reason)
     if (err == ARCHIVE_EOF)
         throw EndOfFile("reached end of archive");
     else if (err != ARCHIVE_OK)
-        throw Error(reason, archive_error_string(this->archive));
+        throw Error(reason, archive_error_string(this->archive.get()));
 }
 
-TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
+TarArchive::TarArchive(Source & source, bool raw)
+    : archive{archive_read_new(), archive_read_free}
+    , source(&source)
+    , buffer(65536)
 {
-    this->archive = archive_read_new();
-    this->source = &source;
-
     if (!raw) {
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
+        archive_read_support_filter_all(archive.get());
+        archive_read_support_format_all(archive.get());
     } else {
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_raw(archive);
-        archive_read_support_format_empty(archive);
+        archive_read_support_filter_all(archive.get());
+        archive_read_support_format_raw(archive.get());
+        archive_read_support_format_empty(archive.get());
     }
-    archive_read_set_option(archive, nullptr, "mac-ext", nullptr);
-    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
-}
-
-
-TarArchive::TarArchive(const Path & path)
-{
-    this->archive = archive_read_new();
-
-    archive_read_support_filter_all(archive);
-    archive_read_support_format_all(archive);
-    archive_read_set_option(archive, nullptr, "mac-ext", nullptr);
+    archive_read_set_option(archive.get(), nullptr, "mac-ext", nullptr);
     check(
-        archive_read_open_filename(archive, requireCString(path), 16384),
+        archive_read_open(
+            archive.get(), (void *) this, callback_open, callback_read, callback_close
+        ),
+        "Failed to open archive (%s)"
+    );
+}
+
+TarArchive::TarArchive(const Path & path) : archive{archive_read_new(), archive_read_free}
+{
+    archive_read_support_filter_all(archive.get());
+    archive_read_support_format_all(archive.get());
+    archive_read_set_option(archive.get(), nullptr, "mac-ext", nullptr);
+    check(
+        archive_read_open_filename(archive.get(), requireCString(path), 16384),
         "failed to open archive: %s"
     );
 }
 
 void TarArchive::close()
 {
-    check(archive_read_close(this->archive), "Failed to close archive (%s)");
+    check(archive_read_close(this->archive.get()), "Failed to close archive (%s)");
 }
 
-TarArchive::~TarArchive()
+namespace {
+
+struct IndexEntry
 {
-    if (this->archive) archive_read_free(this->archive);
+    struct IndexDirectory
+    {
+        std::map<std::string, IndexEntry> entries;
+    };
+
+    struct IndexFile
+    {
+        uint64_t start;
+        uint64_t size;
+    };
+
+    std::variant<IndexDirectory, IndexFile> innerData;
+    int mode;
+};
+
+/** Extract a TAR archive into an in-memory index with data stored on disk,
+ * because fundamentally all we have to do when importing archives to the store
+ * is to sort the entries and then send it into a NAR serializer.
+ *
+ * This deals with macOS performance problems by not materializing the files to
+ * disk in the first place.
+ *
+ * https://git.lix.systems/lix-project/lix/issues/1072
+ */
+struct InMemoryIndex
+{
+    Path tempDir;
+    AutoDelete tempDirDeleter;
+    AutoCloseFD backingFile;
+    std::ofstream writer;
+
+    IndexEntry topLevel;
+
+    InMemoryIndex() : tempDir(createTempDir()), tempDirDeleter(tempDir, true)
+    {
+        backingFile = sys::open(tempDir + "/extracted", O_RDWR | O_TRUNC | O_CLOEXEC);
+    }
+};
+
 }
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
     requireCString(destDir);
 
-    int flags = ARCHIVE_EXTRACT_TIME
-        | ARCHIVE_EXTRACT_SECURE_SYMLINKS
-        | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
+    int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS
+        | ARCHIVE_EXTRACT_SECURE_NODOTDOT | ARCHIVE_EXTRACT_NO_HFS_COMPRESSION;
 
     for (;;) {
         struct archive_entry * entry;
-        int r = archive_read_next_header(archive.archive, &entry);
+        int r = archive_read_next_header(archive.archive.get(), &entry);
         if (r == ARCHIVE_EOF) break;
         auto name = archive_entry_pathname(entry);
         if (!name)
-            throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
+            throw Error(
+                "cannot get archive member name: %s", archive_error_string(archive.archive.get())
+            );
         if (r == ARCHIVE_WARN)
-            printTaggedWarning("%1%", Uncolored(archive_error_string(archive.archive)));
+            printTaggedWarning("%1%", Uncolored(archive_error_string(archive.archive.get())));
         else
             archive.check(r);
 
@@ -125,7 +170,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
                 (destDir + "/" + original_hardlink).c_str());
         }
 
-        archive.check(archive_read_extract(archive.archive, entry, flags));
+        archive.check(archive_read_extract(archive.archive.get(), entry, flags));
     }
 
     archive.close();

lix/libutil/tarfile.hh

@@ -8,7 +8,7 @@
 namespace nix {
 
 struct TarArchive {
-    struct archive * archive;
+    std::unique_ptr<struct archive, decltype(&archive_read_free)> archive;
     Source * source;
     std::vector<unsigned char> buffer;
 
@@ -22,8 +22,6 @@ struct TarArchive {
     TarArchive(const TarArchive &) = delete;
 
     void close();
-
-    ~TarArchive();
 };
 
 kj::Promise<Result<void>> unpackTarfile(AsyncInputStream & source, const Path & destDir);