From c108f339f5bcbaf451b59a1b642c7e17171ad9f3 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 15 Jun 2025 14:47:08 +0200 Subject: [PATCH] libstore: asyncify BinaryCacheStore::getFile Change-Id: If3a1f127470fdaffb0bf79e0692c5d6baf21f18e --- lix/libstore/binary-cache-store.cc | 14 ++++++-------- lix/libstore/binary-cache-store.hh | 2 +- lix/libstore/http-binary-cache-store.cc | 21 ++++++++++++++++++--- lix/libstore/local-binary-cache-store.cc | 11 ++++++++--- lix/libstore/s3-binary-cache-store.cc | 15 +++++++++------ lix/nix/ls.cc | 6 +++++- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc index 75b53f375..7e4731739 100644 --- a/lix/libstore/binary-cache-store.cc +++ b/lix/libstore/binary-cache-store.cc @@ -75,7 +75,7 @@ kj::Promise>> BinaryCacheStore::getFileContents(const std::string & path) try { try { - co_return getFile(path)->drain(); + co_return TRY_AWAIT(TRY_AWAIT(getFile(path))->drain()); } catch (NoSuchBinaryCacheFile &) { co_return std::nullopt; } @@ -372,16 +372,14 @@ try { struct NarFromPath : AsyncInputStream { Stats & stats; - box_ptr file; box_ptr decompressed; uint64_t total; - NarFromPath(Stats & stats, const std::string & method, box_ptr file) + NarFromPath( + Stats & stats, const std::string & method, box_ptr file + ) : stats(stats) - , file(std::move(file)) - , decompressed( - makeDecompressionStream(method, make_box_ptr(*this->file)) - ) + , decompressed(makeDecompressionStream(method, std::move(file))) { } @@ -407,7 +405,7 @@ try { auto & info = *info_; try { - auto file = getFile(info->url); + auto file = TRY_AWAIT(getFile(info->url)); co_return make_box_ptr(stats, info->compression, std::move(file)); } catch (NoSuchBinaryCacheFile & e) { throw SubstituteGone(std::move(e.info())); diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index 4def85392..9f05f9da6 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -87,7 +87,7 @@ public: /** * Dump the contents of the specified file to a sink. */ - virtual box_ptr getFile(const std::string & path) = 0; + virtual kj::Promise>> getFile(const std::string & path) = 0; virtual kj::Promise>> getFileContents(const std::string & path ); diff --git a/lix/libstore/http-binary-cache-store.cc b/lix/libstore/http-binary-cache-store.cc index 39576d740..5050a7f49 100644 --- a/lix/libstore/http-binary-cache-store.cc +++ b/lix/libstore/http-binary-cache-store.cc @@ -3,7 +3,10 @@ #include "lix/libstore/filetransfer.hh" #include "lix/libstore/globals.hh" #include "lix/libstore/nar-info-disk-cache.hh" +#include "lix/libutil/async-io.hh" +#include "lix/libutil/box_ptr.hh" #include "lix/libutil/result.hh" +#include "lix/libutil/serialise.hh" namespace nix { @@ -151,17 +154,29 @@ protected: : cacheUri + "/" + path; } - box_ptr getFile(const std::string & path) override - { + kj::Promise>> getFile(const std::string & path) override + try { checkEnabled(); try { - return getFileTransfer()->download(makeURI(path)).second; + struct HttpFile : AsyncSourceInputStream + { + box_ptr source; + + HttpFile(box_ptr source) + : AsyncSourceInputStream(*source) + , source(std::move(source)) + { + } + }; + return {make_box_ptr(getFileTransfer()->download(makeURI(path)).second)}; } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; } + } catch (...) { + return {result::current_exception()}; } /** diff --git a/lix/libstore/local-binary-cache-store.cc b/lix/libstore/local-binary-cache-store.cc index b8b78be7f..ce29b3dd9 100644 --- a/lix/libstore/local-binary-cache-store.cc +++ b/lix/libstore/local-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "lix/libstore/binary-cache-store.hh" #include "lix/libstore/globals.hh" #include "lix/libstore/nar-info-disk-cache.hh" +#include "lix/libutil/async-io.hh" #include "lix/libutil/result.hh" #include @@ -72,15 +73,19 @@ protected: del.cancel(); } - box_ptr getFile(const std::string & path) override - { + kj::Promise>> getFile(const std::string & path) override + try { try { - return make_box_ptr(readFileSource(binaryCacheDir + "/" + path)); + return { + make_box_ptr(readFileSource(binaryCacheDir + "/" + path)) + }; } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); throw; } + } catch (...) { + return {result::current_exception()}; } kj::Promise> queryAllValidPaths() override diff --git a/lix/libstore/s3-binary-cache-store.cc b/lix/libstore/s3-binary-cache-store.cc index 6efa52cbc..5799b389d 100644 --- a/lix/libstore/s3-binary-cache-store.cc +++ b/lix/libstore/s3-binary-cache-store.cc @@ -5,6 +5,7 @@ #include "lix/libstore/nar-info.hh" #include "lix/libstore/nar-info-disk-cache.hh" #include "lix/libstore/globals.hh" +#include "lix/libutil/async-io.hh" #include "lix/libutil/compression.hh" #include "lix/libstore/filetransfer.hh" #include "lix/libutil/result.hh" @@ -464,8 +465,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore uploadFile(path, istream, mimeType, ""); } - box_ptr getFile(const std::string & path) override - { + kj::Promise>> getFile(const std::string & path) override + try { stats.get++; // FIXME: stream output to sink. @@ -478,13 +479,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, res.data->size(), res.durationMs); - return make_box_ptr( - [](std::string data) -> Generator { + return { + make_box_ptr([](std::string data) -> Generator { co_yield std::span{data.data(), data.size()}; - }(std::move(*res.data)) - ); + }(std::move(*res.data))) + }; } else throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + } catch (...) { + return {result::current_exception()}; } kj::Promise> queryAllValidPaths() override diff --git a/lix/nix/ls.cc b/lix/nix/ls.cc index 4f6de63cd..01f5d7932 100644 --- a/lix/nix/ls.cc +++ b/lix/nix/ls.cc @@ -130,7 +130,11 @@ struct CmdLsStore : StoreCommand, MixLs auto binaryCacheStore = store.try_cast_shared(); if (binaryCacheStore) { const auto [storePath, restPath] = store->toStorePath(path); - auto file = binaryCacheStore->getFile(fmt("%s.ls", storePath.hashPart()))->drain(); + auto file = aio().blockOn( + aio() + .blockOn(binaryCacheStore->getFile(fmt("%s.ls", storePath.hashPart()))) + ->drain() + ); JSON j = json::parse(std::move(file), "a nar content listing"); if (j["version"] == 1) { path = restPath;