guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify BinaryCacheStore::getFile

Browse Source 
Change-Id: If3a1f127470fdaffb0bf79e0692c5d6baf21f18e
This commit is contained in:
eldritch horrors
2025-06-15 13:36:31 +00:00
parent 9f32ab85e8
commit c108f339f5
6 changed files with 47 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/libstore/binary-cache-store.cc
+6 -8
View File
@@ -75,7 +75,7 @@ kj::Promise<Result<std::optional<std::string>>>
BinaryCacheStore::getFileContents(const std::string & path)
try {
try {
co_return getFile(path)->drain();
co_return TRY_AWAIT(TRY_AWAIT(getFile(path))->drain());
} catch (NoSuchBinaryCacheFile &) {
co_return std::nullopt;
}
@@ -372,16 +372,14 @@ try {
struct NarFromPath : AsyncInputStream
{
Stats<std::atomic> & stats;
box_ptr<Source> file;
box_ptr<AsyncInputStream> decompressed;
uint64_t total;
NarFromPath(Stats<std::atomic> & stats, const std::string & method, box_ptr<Source> file)
NarFromPath(
Stats<std::atomic> & stats, const std::string & method, box_ptr<AsyncInputStream> file
)
: stats(stats)
, file(std::move(file))
, decompressed(
makeDecompressionStream(method, make_box_ptr<AsyncSourceInputStream>(*this->file))
)
, decompressed(makeDecompressionStream(method, std::move(file)))
{
}
@@ -407,7 +405,7 @@ try {
auto & info = *info_;
try {
auto file = getFile(info->url);
auto file = TRY_AWAIT(getFile(info->url));
co_return make_box_ptr<NarFromPath>(stats, info->compression, std::move(file));
} catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(std::move(e.info()));
lix/libstore/binary-cache-store.hh
+1 -1
View File
@@ -87,7 +87,7 @@ public:
/**
* Dump the contents of the specified file to a sink.
*/
virtual box_ptr<Source> getFile(const std::string & path) = 0;
virtual kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) = 0;
virtual kj::Promise<Result<std::optional<std::string>>> getFileContents(const std::string & path
);
lix/libstore/http-binary-cache-store.cc
+18 -3
View File
@@ -3,7 +3,10 @@
#include "lix/libstore/filetransfer.hh"
#include "lix/libstore/globals.hh"
#include "lix/libstore/nar-info-disk-cache.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/box_ptr.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/serialise.hh"
namespace nix {
@@ -151,17 +154,29 @@ protected:
: cacheUri + "/" + path;
}
box_ptr<Source> getFile(const std::string & path) override
{
kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) override
try {
checkEnabled();
try {
return getFileTransfer()->download(makeURI(path)).second;
struct HttpFile : AsyncSourceInputStream
{
box_ptr<Source> source;
HttpFile(box_ptr<Source> source)
: AsyncSourceInputStream(*source)
, source(std::move(source))
{
}
};
return {make_box_ptr<HttpFile>(getFileTransfer()->download(makeURI(path)).second)};
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
}
} catch (...) {
return {result::current_exception()};
}
/**
lix/libstore/local-binary-cache-store.cc
+8 -3
View File
@@ -2,6 +2,7 @@
#include "lix/libstore/binary-cache-store.hh"
#include "lix/libstore/globals.hh"
#include "lix/libstore/nar-info-disk-cache.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/result.hh"
#include <atomic>
@@ -72,15 +73,19 @@ protected:
del.cancel();
}
box_ptr<Source> getFile(const std::string & path) override
{
kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) override
try {
try {
return make_box_ptr<GeneratorSource>(readFileSource(binaryCacheDir + "/" + path));
return {
make_box_ptr<AsyncGeneratorInputStream>(readFileSource(binaryCacheDir + "/" + path))
};
} catch (SysError & e) {
if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
throw;
}
} catch (...) {
return {result::current_exception()};
}
kj::Promise<Result<StorePathSet>> queryAllValidPaths() override
lix/libstore/s3-binary-cache-store.cc
+9 -6
View File
@@ -5,6 +5,7 @@
#include "lix/libstore/nar-info.hh"
#include "lix/libstore/nar-info-disk-cache.hh"
#include "lix/libstore/globals.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/compression.hh"
#include "lix/libstore/filetransfer.hh"
#include "lix/libutil/result.hh"
@@ -464,8 +465,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
uploadFile(path, istream, mimeType, "");
}
box_ptr<Source> getFile(const std::string & path) override
{
kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) override
try {
stats.get++;
// FIXME: stream output to sink.
@@ -478,13 +479,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
return make_box_ptr<GeneratorSource>(
[](std::string data) -> Generator<Bytes> {
return {
make_box_ptr<AsyncGeneratorInputStream>([](std::string data) -> Generator<Bytes> {
co_yield std::span{data.data(), data.size()};
}(std::move(*res.data))
);
}(std::move(*res.data)))
};
} else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
} catch (...) {
return {result::current_exception()};
}
kj::Promise<Result<StorePathSet>> queryAllValidPaths() override
lix/nix/ls.cc
+5 -1
View File
@@ -130,7 +130,11 @@ struct CmdLsStore : StoreCommand, MixLs
auto binaryCacheStore = store.try_cast_shared<BinaryCacheStore>();
if (binaryCacheStore) {
const auto [storePath, restPath] = store->toStorePath(path);
auto file = binaryCacheStore->getFile(fmt("%s.ls", storePath.hashPart()))->drain();
auto file = aio().blockOn(
aio()
.blockOn(binaryCacheStore->getFile(fmt("%s.ls", storePath.hashPart())))
->drain()
);
JSON j = json::parse(std::move(file), "a nar content listing");
if (j["version"] == 1) {
path = restPath;