guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libfetchers: asyncify downloadFile, downloadTarball

Browse Source 
Change-Id: I6f6e5b61667efeda8c575ae1f3bd87ee75fb9f9e
This commit is contained in:
eldritch horrors
2025-02-08 17:26:19 +00:00
parent 60f391ef3d
commit 8bfd2d4bb1
8 changed files with 61 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/legacy/nix-channel.cc
+14 -3
View File
@@ -115,7 +115,12 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames)
// We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download.
auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
auto result = aio.blockOn(fetchers::downloadFile(
store,
url,
std::string(baseNameOf(url)),
false
));
auto filename = store->toRealPath(result.storePath);
url = result.effectiveUrl;
@@ -128,11 +133,17 @@ static void update(AsyncIoRoot & aio, const StringSet & channelNames)
if (!unpacked) {
// Download the channel tarball.
std::optional<fetchers::DownloadFileResult> exprs;
try {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
exprs = aio.blockOn(fetchers::downloadFile(
store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false
));
} catch (FileTransferError & e) {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
exprs = aio.blockOn(fetchers::downloadFile(
store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false
));
}
filename = store->toRealPath(exprs->storePath);
}
// Regardless of where it came from, add the expression representing this channel to accumulated expression
exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }");
lix/libcmd/common-eval-args.cc
+2 -2
View File
@@ -192,12 +192,12 @@ kj::Promise<Result<SourcePath>> lookupFileArg(Evaluator & state, std::string_vie
try {
if (EvalSettings::isPseudoUrl(fileArg)) {
auto const url = EvalSettings::resolvePseudoUrl(fileArg);
auto const downloaded = fetchers::downloadTarball(
auto const downloaded = TRY_AWAIT(fetchers::downloadTarball(
state.store,
url,
/* name */ "source",
/* locked */ false
);
));
StorePath const storePath = downloaded.tree.storePath;
co_return CanonPath(state.store->toRealPath(storePath));
} else if (fileArg.starts_with("flake:")) {
lix/libexpr/eval.cc
+2 -2
View File
@@ -2798,8 +2798,8 @@ try {
if (EvalSettings::isPseudoUrl(value)) {
try {
auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
auto storePath = TRY_AWAIT(fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false)).tree.storePath;
res = { store->toRealPath(storePath) };
} catch (FileTransferError & e) {
logWarning({
lix/libexpr/primops/fetchTree.cc
+7 -4
View File
@@ -270,10 +270,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// TODO: fetching may fail, yet the path may be substitutable.
// https://github.com/NixOS/nix/issues/4313
auto storePath =
unpack
? fetchers::downloadTarball(state.ctx.store, *url, name, (bool) expectedHash).tree.storePath
: fetchers::downloadFile(state.ctx.store, *url, name, (bool) expectedHash).storePath;
auto storePath = unpack
? state.aio
.blockOn(fetchers::downloadTarball(state.ctx.store, *url, name, (bool) expectedHash))
.tree.storePath
: state.aio
.blockOn(fetchers::downloadFile(state.ctx.store, *url, name, (bool) expectedHash))
.storePath;
if (expectedHash) {
auto hash = unpack
lix/libfetchers/fetchers.hh
+2 -2
View File
@@ -209,7 +209,7 @@ struct DownloadFileResult
std::optional<std::string> immutableUrl;
};
DownloadFileResult downloadFile(
kj::Promise<Result<DownloadFileResult>> downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
@@ -223,7 +223,7 @@ struct DownloadTarballResult
std::optional<std::string> immutableUrl;
};
DownloadTarballResult downloadTarball(
kj::Promise<Result<DownloadTarballResult>> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
lix/libfetchers/github.cc
+14 -11
View File
@@ -225,7 +225,8 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
auto result = downloadTarball(store, url.url, input.getName(), true, url.headers);
auto result =
TRY_AWAIT(downloadTarball(store, url.url, input.getName(), true, url.headers));
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
@@ -277,10 +278,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto json = nlohmann::json::parse(readFile(store->toRealPath(
TRY_AWAIT(downloadFile(store, url, "source", false, headers)).storePath
)));
auto rev = Hash::parseAny(std::string { json["sha"] }, HashType::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
co_return rev;
@@ -359,10 +359,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto json = nlohmann::json::parse(readFile(store->toRealPath(
TRY_AWAIT(downloadFile(store, url, "source", false, headers)).storePath
)));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
auto rev = Hash::parseAny(std::string(json[0]["id"]), HashType::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
@@ -434,7 +433,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string refUri;
if (ref == "HEAD") {
auto file = store->toRealPath(
downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath);
TRY_AWAIT(downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers))
.storePath
);
std::ifstream is(file);
std::string line;
getline(is, line);
@@ -450,7 +451,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::regex refRegex(refUri);
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
TRY_AWAIT(downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers))
.storePath
);
std::ifstream is(file);
std::string line;
lix/libfetchers/registry.cc
+2 -1
View File
@@ -183,7 +183,8 @@ try {
path
);
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
auto storePath =
TRY_AWAIT(downloadFile(store, path, "flake-registry.json", false)).storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
TRY_AWAIT(
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json")
lix/libfetchers/tarball.cc
+18 -12
View File
@@ -5,6 +5,7 @@
#include "lix/libfetchers/builtin-fetchers.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libutil/archive.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/tarfile.hh"
#include "lix/libstore/temporary-dir.hh"
#include "lix/libutil/types.hh"
@@ -12,13 +13,13 @@
namespace nix::fetchers {
DownloadFileResult downloadFile(
kj::Promise<Result<DownloadFileResult>> downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
Headers headers)
{
try {
// FIXME: check store
Attrs inAttrs({
@@ -40,7 +41,7 @@ DownloadFileResult downloadFile(
};
if (cached && !cached->expired)
return useCached();
co_return useCached();
if (cached)
headers.emplace_back("If-None-Match", getStrAttr(cached->infoAttrs, "etag"));
@@ -53,7 +54,7 @@ DownloadFileResult downloadFile(
} catch (FileTransferError & e) {
if (cached) {
warn("%s; using cached version", e.msg());
return useCached();
co_return useCached();
} else
throw;
}
@@ -111,21 +112,23 @@ DownloadFileResult downloadFile(
*storePath,
locked);
return {
co_return DownloadFileResult{
.storePath = std::move(*storePath),
.etag = res.etag,
.effectiveUrl = res.effectiveUri,
.immutableUrl = res.immutableUrl,
};
} catch (...) {
co_return result::current_exception();
}
DownloadTarballResult downloadTarball(
kj::Promise<Result<DownloadTarballResult>> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers)
{
try {
Attrs inAttrs({
{"type", "tarball"},
{"url", url},
@@ -135,13 +138,13 @@ DownloadTarballResult downloadTarball(
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
return {
co_return DownloadTarballResult{
.tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
.lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
};
auto res = downloadFile(store, url, name, locked, headers);
auto res = TRY_AWAIT(downloadFile(store, url, name, locked, headers));
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
@@ -176,11 +179,13 @@ DownloadTarballResult downloadTarball(
*unpackedStorePath,
locked);
return {
co_return DownloadTarballResult{
.tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
.lastModified = lastModified,
.immutableUrl = res.immutableUrl,
};
} catch (...) {
co_return result::current_exception();
}
// An input scheme corresponding to a curl-downloadable resource.
@@ -268,7 +273,8 @@ struct FileInputScheme : CurlInputScheme
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & input) override
try {
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
auto file =
TRY_AWAIT(downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false));
co_return {std::move(file.storePath), input};
} catch (...) {
co_return result::current_exception();
@@ -294,7 +300,7 @@ struct TarballInputScheme : CurlInputScheme
try {
Input input(_input);
auto url = getStrAttr(input.attrs, "url");
auto result = downloadTarball(store, url, input.getName(), false);
auto result = TRY_AWAIT(downloadTarball(store, url, input.getName(), false));
if (result.immutableUrl) {
auto immutableInput = Input::fromURL(*result.immutableUrl);