guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libfetchers: asyncify InputScheme::fetch

Browse Source 
Change-Id: I8d4d8d1d4137635ade3e4b05b5d4cd3ecb3390c9
This commit is contained in:
eldritch horrors
2025-02-08 17:01:31 +00:00
parent b60314f32e
commit 6ffa9f4298
8 changed files with 74 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/libfetchers/fetchers.cc
+21 -14
View File
@@ -149,20 +149,27 @@ try {
}
}
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
// *sighs*, we print the URL without query params, rather than the full URL
// because the Nixpkgs fileset lib tests assume that fetching shallow and
// non-shallow prints exactly the same stderr...
ParsedURL withoutParams = this->toURL();
withoutParams.query.clear();
printInfo("fetching %s input '%s'", this->getType(), withoutParams.to_string());
try {
return scheme->fetch(store, *this);
} catch (Error & e) {
e.addTrace({}, "while fetching the input '%s'", to_string());
throw;
}
}();
auto [storePath, input] = TRY_AWAIT(
[](const Input & self,
ref<Store> store) -> kj::Promise<Result<std::pair<StorePath, Input>>> {
try {
// *sighs*, we print the URL without query params, rather than the full URL
// because the Nixpkgs fileset lib tests assume that fetching shallow and
// non-shallow prints exactly the same stderr...
ParsedURL withoutParams = self.toURL();
withoutParams.query.clear();
printInfo("fetching %s input '%s'", self.getType(), withoutParams.to_string());
try {
co_return TRY_AWAIT(self.scheme->fetch(store, self));
} catch (Error & e) {
e.addTrace({}, "while fetching the input '%s'", self.to_string());
throw;
}
} catch (...) {
co_return result::current_exception();
}
}(*this, store)
);
Tree tree {
.actualPath = store->toRealPath(storePath),
lix/libfetchers/fetchers.hh
+2 -1
View File
@@ -162,7 +162,8 @@ struct InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const;
virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
virtual kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & input) = 0;
protected:
void emplaceURLQueryIntoAttrs(
lix/libfetchers/git.cc
+12 -8
View File
@@ -1,3 +1,4 @@
#include "lix/libutil/async.hh"
#include "lix/libutil/error.hh"
#include "lix/libfetchers/fetchers.hh"
#include "lix/libfetchers/cache.hh"
@@ -444,8 +445,9 @@ struct GitInputScheme : InputScheme
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & _input) override
try {
Input input(_input);
auto gitDir = ".git";
@@ -490,7 +492,7 @@ struct GitInputScheme : InputScheme
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl(input);
@@ -501,7 +503,7 @@ struct GitInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal) {
auto workdirInfo = getWorkdirInfo(input, actualUrl);
if (!workdirInfo.clean) {
return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
co_return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
}
}
@@ -549,7 +551,7 @@ struct GitInputScheme : InputScheme
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
}
}
@@ -558,7 +560,7 @@ struct GitInputScheme : InputScheme
gitDir = ".";
createDirs(dirOf(cacheDir));
PathLock cacheDirLock = lockPath(cacheDir + ".lock");
PathLock cacheDirLock = TRY_AWAIT(lockPathAsync(cacheDir + ".lock"));
if (!pathExists(cacheDir)) {
runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
@@ -685,7 +687,7 @@ struct GitInputScheme : InputScheme
/* Now that we know the ref, check again whether we have it in
the store. */
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
@@ -792,7 +794,9 @@ struct GitInputScheme : InputScheme
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
co_return makeResult(infoAttrs, std::move(storePath));
} catch (...) {
co_return result::current_exception();
}
};
lix/libfetchers/github.cc
+6 -3
View File
@@ -208,8 +208,9 @@ struct GitArchiveInputScheme : InputScheme
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & _input) override
try {
Input input(_input);
if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
@@ -226,7 +227,9 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
return {result.tree.storePath, input};
co_return {result.tree.storePath, input};
} catch (...) {
co_return result::current_exception();
}
};
lix/libfetchers/indirect.cc
+5 -2
View File
@@ -102,9 +102,12 @@ struct IndirectInputScheme : InputScheme
return input;
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & input) override
try {
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
} catch (...) {
return {result::current_exception()};
}
};
lix/libfetchers/mercurial.cc
+10 -7
View File
@@ -150,8 +150,9 @@ struct MercurialInputScheme : InputScheme
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & _input) override
try {
Input input(_input);
auto name = input.getName();
@@ -202,7 +203,7 @@ struct MercurialInputScheme : InputScheme
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashType::SHA256, filter);
return {std::move(storePath), input};
co_return {std::move(storePath), input};
}
auto tokens = tokenizeString<std::vector<std::string>>(
@@ -243,7 +244,7 @@ struct MercurialInputScheme : InputScheme
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
}
auto revOrRef = input.getRev() ? fmt("id(%s)", input.getRev()->gitRev()) : *input.getRef();
@@ -259,7 +260,7 @@ struct MercurialInputScheme : InputScheme
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
}
}
@@ -302,7 +303,7 @@ struct MercurialInputScheme : InputScheme
input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
co_return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
@@ -333,7 +334,9 @@ struct MercurialInputScheme : InputScheme
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
co_return makeResult(infoAttrs, std::move(storePath));
} catch (...) {
co_return result::current_exception();
}
};
lix/libfetchers/path.cc
+6 -3
View File
@@ -96,8 +96,9 @@ struct PathInputScheme : InputScheme
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & _input) override
try {
Input input(_input);
std::string absPath;
auto path = getStrAttr(input.attrs, "path");
@@ -136,7 +137,9 @@ struct PathInputScheme : InputScheme
}
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
return {std::move(*storePath), input};
co_return {std::move(*storePath), input};
} catch (...) {
co_return result::current_exception();
}
};
lix/libfetchers/tarball.cc
+12 -6
View File
@@ -265,10 +265,13 @@ struct FileInputScheme : CurlInputScheme
: (!requireTree && !hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & input) override
try {
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
return {std::move(file.storePath), input};
co_return {std::move(file.storePath), input};
} catch (...) {
co_return result::current_exception();
}
};
@@ -286,8 +289,9 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
kj::Promise<Result<std::pair<StorePath, Input>>>
fetch(ref<Store> store, const Input & _input) override
try {
Input input(_input);
auto url = getStrAttr(input.attrs, "url");
auto result = downloadTarball(store, url, input.getName(), false);
@@ -304,7 +308,9 @@ struct TarballInputScheme : CurlInputScheme
if (result.lastModified && !input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
return {result.tree.storePath, std::move(input)};
co_return {result.tree.storePath, std::move(input)};
} catch (...) {
co_return result::current_exception();
}
};