From 6ffa9f4298f554cace74b2526040eb3e2025b22c Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 2 Feb 2025 20:07:35 +0100 Subject: [PATCH] libfetchers: asyncify InputScheme::fetch Change-Id: I8d4d8d1d4137635ade3e4b05b5d4cd3ecb3390c9 --- lix/libfetchers/fetchers.cc | 35 +++++++++++++++++++++-------------- lix/libfetchers/fetchers.hh | 3 ++- lix/libfetchers/git.cc | 20 ++++++++++++-------- lix/libfetchers/github.cc | 9 ++++++--- lix/libfetchers/indirect.cc | 7 +++++-- lix/libfetchers/mercurial.cc | 17 ++++++++++------- lix/libfetchers/path.cc | 9 ++++++--- lix/libfetchers/tarball.cc | 18 ++++++++++++------ 8 files changed, 74 insertions(+), 44 deletions(-) diff --git a/lix/libfetchers/fetchers.cc b/lix/libfetchers/fetchers.cc index 703e30bce..dcd8a8080 100644 --- a/lix/libfetchers/fetchers.cc +++ b/lix/libfetchers/fetchers.cc @@ -149,20 +149,27 @@ try { } } - auto [storePath, input] = [&]() -> std::pair { - // *sighs*, we print the URL without query params, rather than the full URL - // because the Nixpkgs fileset lib tests assume that fetching shallow and - // non-shallow prints exactly the same stderr... - ParsedURL withoutParams = this->toURL(); - withoutParams.query.clear(); - printInfo("fetching %s input '%s'", this->getType(), withoutParams.to_string()); - try { - return scheme->fetch(store, *this); - } catch (Error & e) { - e.addTrace({}, "while fetching the input '%s'", to_string()); - throw; - } - }(); + auto [storePath, input] = TRY_AWAIT( + [](const Input & self, + ref store) -> kj::Promise>> { + try { + // *sighs*, we print the URL without query params, rather than the full URL + // because the Nixpkgs fileset lib tests assume that fetching shallow and + // non-shallow prints exactly the same stderr... + ParsedURL withoutParams = self.toURL(); + withoutParams.query.clear(); + printInfo("fetching %s input '%s'", self.getType(), withoutParams.to_string()); + try { + co_return TRY_AWAIT(self.scheme->fetch(store, self)); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", self.to_string()); + throw; + } + } catch (...) { + co_return result::current_exception(); + } + }(*this, store) + ); Tree tree { .actualPath = store->toRealPath(storePath), diff --git a/lix/libfetchers/fetchers.hh b/lix/libfetchers/fetchers.hh index 4393b78c1..5ceaf057c 100644 --- a/lix/libfetchers/fetchers.hh +++ b/lix/libfetchers/fetchers.hh @@ -162,7 +162,8 @@ struct InputScheme std::string_view contents, std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual kj::Promise>> + fetch(ref store, const Input & input) = 0; protected: void emplaceURLQueryIntoAttrs( diff --git a/lix/libfetchers/git.cc b/lix/libfetchers/git.cc index 80ad1628c..7240fba07 100644 --- a/lix/libfetchers/git.cc +++ b/lix/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "lix/libutil/async.hh" #include "lix/libutil/error.hh" #include "lix/libfetchers/fetchers.hh" #include "lix/libfetchers/cache.hh" @@ -444,8 +445,9 @@ struct GitInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override - { + kj::Promise>> + fetch(ref store, const Input & _input) override + try { Input input(_input); auto gitDir = ".git"; @@ -490,7 +492,7 @@ struct GitInputScheme : InputScheme if (input.getRev()) { if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); } auto [isLocal, actualUrl_] = getActualUrl(input); @@ -501,7 +503,7 @@ struct GitInputScheme : InputScheme if (!input.getRef() && !input.getRev() && isLocal) { auto workdirInfo = getWorkdirInfo(input, actualUrl); if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); + co_return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } @@ -549,7 +551,7 @@ struct GitInputScheme : InputScheme auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); } } @@ -558,7 +560,7 @@ struct GitInputScheme : InputScheme gitDir = "."; createDirs(dirOf(cacheDir)); - PathLock cacheDirLock = lockPath(cacheDir + ".lock"); + PathLock cacheDirLock = TRY_AWAIT(lockPathAsync(cacheDir + ".lock")); if (!pathExists(cacheDir)) { runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); @@ -685,7 +687,7 @@ struct GitInputScheme : InputScheme /* Now that we know the ref, check again whether we have it in the store. */ if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -792,7 +794,9 @@ struct GitInputScheme : InputScheme storePath, true); - return makeResult(infoAttrs, std::move(storePath)); + co_return makeResult(infoAttrs, std::move(storePath)); + } catch (...) { + co_return result::current_exception(); } }; diff --git a/lix/libfetchers/github.cc b/lix/libfetchers/github.cc index cc01f0528..d3ac4e7dd 100644 --- a/lix/libfetchers/github.cc +++ b/lix/libfetchers/github.cc @@ -208,8 +208,9 @@ struct GitArchiveInputScheme : InputScheme virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair fetch(ref store, const Input & _input) override - { + kj::Promise>> + fetch(ref store, const Input & _input) override + try { Input input(_input); if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); @@ -226,7 +227,9 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, input}; + co_return {result.tree.storePath, input}; + } catch (...) { + co_return result::current_exception(); } }; diff --git a/lix/libfetchers/indirect.cc b/lix/libfetchers/indirect.cc index 3f4e30a4c..9f7678834 100644 --- a/lix/libfetchers/indirect.cc +++ b/lix/libfetchers/indirect.cc @@ -102,9 +102,12 @@ struct IndirectInputScheme : InputScheme return input; } - std::pair fetch(ref store, const Input & input) override - { + kj::Promise>> + fetch(ref store, const Input & input) override + try { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } catch (...) { + return {result::current_exception()}; } }; diff --git a/lix/libfetchers/mercurial.cc b/lix/libfetchers/mercurial.cc index bd3037e3c..9e20c29a7 100644 --- a/lix/libfetchers/mercurial.cc +++ b/lix/libfetchers/mercurial.cc @@ -150,8 +150,9 @@ struct MercurialInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override - { + kj::Promise>> + fetch(ref store, const Input & _input) override + try { Input input(_input); auto name = input.getName(); @@ -202,7 +203,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashType::SHA256, filter); - return {std::move(storePath), input}; + co_return {std::move(storePath), input}; } auto tokens = tokenizeString>( @@ -243,7 +244,7 @@ struct MercurialInputScheme : InputScheme if (input.getRev()) { if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); } auto revOrRef = input.getRev() ? fmt("id(%s)", input.getRev()->gitRev()) : *input.getRef(); @@ -259,7 +260,7 @@ struct MercurialInputScheme : InputScheme auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); } } @@ -302,7 +303,7 @@ struct MercurialInputScheme : InputScheme input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + co_return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -333,7 +334,9 @@ struct MercurialInputScheme : InputScheme storePath, true); - return makeResult(infoAttrs, std::move(storePath)); + co_return makeResult(infoAttrs, std::move(storePath)); + } catch (...) { + co_return result::current_exception(); } }; diff --git a/lix/libfetchers/path.cc b/lix/libfetchers/path.cc index 97f625d9d..a6d717528 100644 --- a/lix/libfetchers/path.cc +++ b/lix/libfetchers/path.cc @@ -96,8 +96,9 @@ struct PathInputScheme : InputScheme throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } - std::pair fetch(ref store, const Input & _input) override - { + kj::Promise>> + fetch(ref store, const Input & _input) override + try { Input input(_input); std::string absPath; auto path = getStrAttr(input.attrs, "path"); @@ -136,7 +137,9 @@ struct PathInputScheme : InputScheme } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {std::move(*storePath), input}; + co_return {std::move(*storePath), input}; + } catch (...) { + co_return result::current_exception(); } }; diff --git a/lix/libfetchers/tarball.cc b/lix/libfetchers/tarball.cc index 2b208c99d..811b45f14 100644 --- a/lix/libfetchers/tarball.cc +++ b/lix/libfetchers/tarball.cc @@ -265,10 +265,13 @@ struct FileInputScheme : CurlInputScheme : (!requireTree && !hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override - { + kj::Promise>> + fetch(ref store, const Input & input) override + try { auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); - return {std::move(file.storePath), input}; + co_return {std::move(file.storePath), input}; + } catch (...) { + co_return result::current_exception(); } }; @@ -286,8 +289,9 @@ struct TarballInputScheme : CurlInputScheme : (requireTree || hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & _input) override - { + kj::Promise>> + fetch(ref store, const Input & _input) override + try { Input input(_input); auto url = getStrAttr(input.attrs, "url"); auto result = downloadTarball(store, url, input.getName(), false); @@ -304,7 +308,9 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, std::move(input)}; + co_return {result.tree.storePath, std::move(input)}; + } catch (...) { + co_return result::current_exception(); } };