From 8bfd2d4bb125d93046ee56a4312fba1e239d019f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 6 Feb 2025 23:48:21 +0100 Subject: [PATCH] libfetchers: asyncify downloadFile, downloadTarball Change-Id: I6f6e5b61667efeda8c575ae1f3bd87ee75fb9f9e --- lix/legacy/nix-channel.cc | 17 ++++++++++++++--- lix/libcmd/common-eval-args.cc | 4 ++-- lix/libexpr/eval.cc | 4 ++-- lix/libexpr/primops/fetchTree.cc | 11 +++++++---- lix/libfetchers/fetchers.hh | 4 ++-- lix/libfetchers/github.cc | 25 ++++++++++++++----------- lix/libfetchers/registry.cc | 3 ++- lix/libfetchers/tarball.cc | 30 ++++++++++++++++++------------ 8 files changed, 61 insertions(+), 37 deletions(-) diff --git a/lix/legacy/nix-channel.cc b/lix/legacy/nix-channel.cc index 7d24a7d67..c994e8583 100644 --- a/lix/legacy/nix-channel.cc +++ b/lix/legacy/nix-channel.cc @@ -115,7 +115,12 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto result = aio.blockOn(fetchers::downloadFile( + store, + url, + std::string(baseNameOf(url)), + false + )); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -128,11 +133,17 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. + std::optional exprs; try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + exprs = aio.blockOn(fetchers::downloadFile( + store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false + )); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + exprs = aio.blockOn(fetchers::downloadFile( + store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false + )); } + filename = store->toRealPath(exprs->storePath); } // Regardless of where it came from, add the expression representing this channel to accumulated expression exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); diff --git a/lix/libcmd/common-eval-args.cc b/lix/libcmd/common-eval-args.cc index 3c7556fb0..2204e104a 100644 --- a/lix/libcmd/common-eval-args.cc +++ b/lix/libcmd/common-eval-args.cc @@ -192,12 +192,12 @@ kj::Promise> lookupFileArg(Evaluator & state, std::string_vie try { if (EvalSettings::isPseudoUrl(fileArg)) { auto const url = EvalSettings::resolvePseudoUrl(fileArg); - auto const downloaded = fetchers::downloadTarball( + auto const downloaded = TRY_AWAIT(fetchers::downloadTarball( state.store, url, /* name */ "source", /* locked */ false - ); + )); StorePath const storePath = downloaded.tree.storePath; co_return CanonPath(state.store->toRealPath(storePath)); } else if (fileArg.starts_with("flake:")) { diff --git a/lix/libexpr/eval.cc b/lix/libexpr/eval.cc index 4d2ae3aa3..0fea5a27f 100644 --- a/lix/libexpr/eval.cc +++ b/lix/libexpr/eval.cc @@ -2798,8 +2798,8 @@ try { if (EvalSettings::isPseudoUrl(value)) { try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + auto storePath = TRY_AWAIT(fetchers::downloadTarball( + store, EvalSettings::resolvePseudoUrl(value), "source", false)).tree.storePath; res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ diff --git a/lix/libexpr/primops/fetchTree.cc b/lix/libexpr/primops/fetchTree.cc index e1c680f82..61d3fb9c0 100644 --- a/lix/libexpr/primops/fetchTree.cc +++ b/lix/libexpr/primops/fetchTree.cc @@ -270,10 +270,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // TODO: fetching may fail, yet the path may be substitutable. // https://github.com/NixOS/nix/issues/4313 - auto storePath = - unpack - ? fetchers::downloadTarball(state.ctx.store, *url, name, (bool) expectedHash).tree.storePath - : fetchers::downloadFile(state.ctx.store, *url, name, (bool) expectedHash).storePath; + auto storePath = unpack + ? state.aio + .blockOn(fetchers::downloadTarball(state.ctx.store, *url, name, (bool) expectedHash)) + .tree.storePath + : state.aio + .blockOn(fetchers::downloadFile(state.ctx.store, *url, name, (bool) expectedHash)) + .storePath; if (expectedHash) { auto hash = unpack diff --git a/lix/libfetchers/fetchers.hh b/lix/libfetchers/fetchers.hh index 5ceaf057c..f40389160 100644 --- a/lix/libfetchers/fetchers.hh +++ b/lix/libfetchers/fetchers.hh @@ -209,7 +209,7 @@ struct DownloadFileResult std::optional immutableUrl; }; -DownloadFileResult downloadFile( +kj::Promise> downloadFile( ref store, const std::string & url, const std::string & name, @@ -223,7 +223,7 @@ struct DownloadTarballResult std::optional immutableUrl; }; -DownloadTarballResult downloadTarball( +kj::Promise> downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/lix/libfetchers/github.cc b/lix/libfetchers/github.cc index 167c6fe6a..66d8ae3a0 100644 --- a/lix/libfetchers/github.cc +++ b/lix/libfetchers/github.cc @@ -225,7 +225,8 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto result = downloadTarball(store, url.url, input.getName(), true, url.headers); + auto result = + TRY_AWAIT(downloadTarball(store, url.url, input.getName(), true, url.headers)); input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); @@ -277,10 +278,9 @@ struct GitHubInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + auto json = nlohmann::json::parse(readFile(store->toRealPath( + TRY_AWAIT(downloadFile(store, url, "source", false, headers)).storePath + ))); auto rev = Hash::parseAny(std::string { json["sha"] }, HashType::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); co_return rev; @@ -359,10 +359,9 @@ struct GitLabInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + auto json = nlohmann::json::parse(readFile(store->toRealPath( + TRY_AWAIT(downloadFile(store, url, "source", false, headers)).storePath + ))); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { auto rev = Hash::parseAny(std::string(json[0]["id"]), HashType::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); @@ -434,7 +433,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + TRY_AWAIT(downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers)) + .storePath + ); std::ifstream is(file); std::string line; getline(is, line); @@ -450,7 +451,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + TRY_AWAIT(downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers)) + .storePath + ); std::ifstream is(file); std::string line; diff --git a/lix/libfetchers/registry.cc b/lix/libfetchers/registry.cc index bb6bc4338..42c57f69d 100644 --- a/lix/libfetchers/registry.cc +++ b/lix/libfetchers/registry.cc @@ -183,7 +183,8 @@ try { path ); - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = + TRY_AWAIT(downloadFile(store, path, "flake-registry.json", false)).storePath; if (auto store2 = store.dynamic_pointer_cast()) { TRY_AWAIT( store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json") diff --git a/lix/libfetchers/tarball.cc b/lix/libfetchers/tarball.cc index 811b45f14..1d05a4642 100644 --- a/lix/libfetchers/tarball.cc +++ b/lix/libfetchers/tarball.cc @@ -5,6 +5,7 @@ #include "lix/libfetchers/builtin-fetchers.hh" #include "lix/libstore/store-api.hh" #include "lix/libutil/archive.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/tarfile.hh" #include "lix/libstore/temporary-dir.hh" #include "lix/libutil/types.hh" @@ -12,13 +13,13 @@ namespace nix::fetchers { -DownloadFileResult downloadFile( +kj::Promise> downloadFile( ref store, const std::string & url, const std::string & name, bool locked, Headers headers) -{ +try { // FIXME: check store Attrs inAttrs({ @@ -40,7 +41,7 @@ DownloadFileResult downloadFile( }; if (cached && !cached->expired) - return useCached(); + co_return useCached(); if (cached) headers.emplace_back("If-None-Match", getStrAttr(cached->infoAttrs, "etag")); @@ -53,7 +54,7 @@ DownloadFileResult downloadFile( } catch (FileTransferError & e) { if (cached) { warn("%s; using cached version", e.msg()); - return useCached(); + co_return useCached(); } else throw; } @@ -111,21 +112,23 @@ DownloadFileResult downloadFile( *storePath, locked); - return { + co_return DownloadFileResult{ .storePath = std::move(*storePath), .etag = res.etag, .effectiveUrl = res.effectiveUri, .immutableUrl = res.immutableUrl, }; +} catch (...) { + co_return result::current_exception(); } -DownloadTarballResult downloadTarball( +kj::Promise> downloadTarball( ref store, const std::string & url, const std::string & name, bool locked, const Headers & headers) -{ +try { Attrs inAttrs({ {"type", "tarball"}, {"url", url}, @@ -135,13 +138,13 @@ DownloadTarballResult downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return { + co_return DownloadTarballResult{ .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; - auto res = downloadFile(store, url, name, locked, headers); + auto res = TRY_AWAIT(downloadFile(store, url, name, locked, headers)); std::optional unpackedStorePath; time_t lastModified; @@ -176,11 +179,13 @@ DownloadTarballResult downloadTarball( *unpackedStorePath, locked); - return { + co_return DownloadTarballResult{ .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; +} catch (...) { + co_return result::current_exception(); } // An input scheme corresponding to a curl-downloadable resource. @@ -268,7 +273,8 @@ struct FileInputScheme : CurlInputScheme kj::Promise>> fetch(ref store, const Input & input) override try { - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + auto file = + TRY_AWAIT(downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false)); co_return {std::move(file.storePath), input}; } catch (...) { co_return result::current_exception(); @@ -294,7 +300,7 @@ struct TarballInputScheme : CurlInputScheme try { Input input(_input); auto url = getStrAttr(input.attrs, "url"); - auto result = downloadTarball(store, url, input.getName(), false); + auto result = TRY_AWAIT(downloadTarball(store, url, input.getName(), false)); if (result.immutableUrl) { auto immutableInput = Input::fromURL(*result.immutableUrl);