guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libfetchers: asyncify some git/hg internals

Browse Source 
Change-Id: I3aea82173a610dc9584e956aa74d3e3ad06bfe68
This commit is contained in:
eldritch horrors
2025-09-12 11:52:03 +00:00
parent 9b01455ada
commit b966d2e53b
2 changed files with 91 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/libfetchers/git.cc
+38 -24
View File
@@ -67,15 +67,17 @@ Path getCachePath(std::string_view key)
//
// ref: refs/heads/main HEAD
// ...
std::optional<std::string> readHead(const Path & path)
{
static kj::Promise<Result<std::optional<std::string>>> readHead(const Path & path)
try {
auto [status, output] = runProgram(RunOptions {
.program = "git",
// FIXME: use 'HEAD' to avoid returning all refs
.args = {"ls-remote", "--symref", path},
.isInteractive = true,
});
if (status != 0) return std::nullopt;
if (status != 0) {
co_return std::nullopt;
}
std::string_view line = output;
line = line.substr(0, line.find("\n"));
@@ -88,27 +90,32 @@ std::optional<std::string> readHead(const Path & path)
debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path);
break;
}
return parseResult->target;
co_return parseResult->target;
}
return std::nullopt;
co_return std::nullopt;
} catch (...) {
co_return result::current_exception();
}
// Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
{
static kj::Promise<Result<bool>>
storeCachedHead(const std::string & actualUrl, const std::string & headRef)
try {
Path cacheDir = getCachePath(actualUrl);
try {
runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
if (!WIFEXITED(e.status)) throw;
return false;
co_return false;
}
/* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
return true;
co_return true;
} catch (...) {
co_return result::current_exception();
}
std::optional<std::string> readHeadCached(const std::string & actualUrl)
{
static kj::Promise<Result<std::optional<std::string>>> readHeadCached(const std::string & actualUrl)
try {
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl);
@@ -118,18 +125,20 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
struct stat st;
std::optional<std::string> cachedRef;
if (stat(headRefFile.c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
cachedRef = TRY_AWAIT(readHead(cacheDir));
if (cachedRef != std::nullopt &&
*cachedRef != gitInitialBranch &&
isCacheFileWithinTtl(now, st))
{
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
co_return cachedRef;
}
}
auto ref = readHead(actualUrl);
if (ref) return ref;
auto ref = TRY_AWAIT(readHead(actualUrl));
if (ref) {
co_return ref;
}
if (cachedRef) {
// If the cached git ref is expired in fetch() below, and the 'git fetch'
@@ -141,10 +150,12 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
actualUrl,
*cachedRef
);
return cachedRef;
co_return cachedRef;
}
return std::nullopt;
co_return std::nullopt;
} catch (...) {
co_return result::current_exception();
}
bool isNotDotGitDirectory(const Path & path)
@@ -159,8 +170,8 @@ struct WorkdirInfo
};
// Returns whether a git workdir is clean and has commits.
WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
{
static kj::Promise<Result<WorkdirInfo>> getWorkdirInfo(const Input & input, const Path & workdir)
try {
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string gitDir(".git");
@@ -220,7 +231,9 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
return WorkdirInfo { .clean = clean, .hasHead = hasHead };
co_return WorkdirInfo{.clean = clean, .hasHead = hasHead};
} catch (...) {
co_return result::current_exception();
}
static kj::Promise<Result<std::pair<StorePath, Input>>> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
@@ -541,7 +554,7 @@ struct GitInputScheme : InputScheme
/* If this is a local directory and no ref or revision is given,
allow fetching directly from a dirty workdir. */
if (!input.getRef() && !input.getRev() && isLocal) {
auto workdirInfo = getWorkdirInfo(input, actualUrl);
auto workdirInfo = TRY_AWAIT(getWorkdirInfo(input, actualUrl));
if (!workdirInfo.clean) {
co_return TRY_AWAIT(fetchFromWorkdir(store, input, actualUrl, workdirInfo));
}
@@ -557,7 +570,7 @@ struct GitInputScheme : InputScheme
if (isLocal) {
if (!input.getRef()) {
auto head = readHead(actualUrl);
auto head = TRY_AWAIT(readHead(actualUrl));
if (!head) {
printTaggedWarning(
"could not read HEAD ref from repo at '%s', using 'master'", actualUrl
@@ -576,7 +589,7 @@ struct GitInputScheme : InputScheme
} else {
const bool useHeadRef = !input.getRef();
if (useHeadRef) {
auto head = readHeadCached(actualUrl);
auto head = TRY_AWAIT(readHeadCached(actualUrl));
if (!head) {
printTaggedWarning(
"could not read HEAD ref from repo at '%s', using 'master'", actualUrl
@@ -715,10 +728,11 @@ struct GitInputScheme : InputScheme
printTaggedWarning(
"could not update mtime for file '%s': %s", localRefFile, strerror(errno)
);
if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
if (useHeadRef && !TRY_AWAIT(storeCachedHead(actualUrl, *input.getRef()))) {
printTaggedWarning(
"could not update cached head '%s' for '%s'", *input.getRef(), actualUrl
);
}
}
if (!input.getRev())
lix/libfetchers/mercurial.cc
+53 -23
View File
@@ -33,8 +33,8 @@ static RunOptions hgOptions(const Strings & args)
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
static std::string runHg(const Strings & args)
{
static kj::Promise<Result<std::string>> runHg(const Strings & args)
try {
RunOptions opts = hgOptions(args);
auto res = runProgram(std::move(opts));
@@ -42,7 +42,9 @@ static std::string runHg(const Strings & args)
if (!statusOk(res.first))
throw ExecError(res.first, "hg %1%", statusToString(res.first));
return res.second;
co_return res.second;
} catch (...) {
co_return result::current_exception();
}
static const std::set<std::string> allowedMercurialAttrs = {
@@ -145,12 +147,11 @@ struct MercurialInputScheme : InputScheme
writeFile(absPath.abs(), contents);
// FIXME: shut up if file is already tracked.
runHg(
{ "add", absPath.abs() });
TRY_AWAIT(runHg({"add", absPath.abs()}));
if (commitMsg)
runHg(
{ "commit", absPath.abs(), "-m", *commitMsg });
if (commitMsg) {
TRY_AWAIT(runHg({"commit", absPath.abs(), "-m", *commitMsg}));
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
@@ -179,7 +180,9 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
bool clean =
TRY_AWAIT(runHg({"status", "-R", actualUrl, "--modified", "--added", "--removed"}))
== "";
if (!clean) {
@@ -192,10 +195,23 @@ struct MercurialInputScheme : InputScheme
if (fetchSettings.warnDirty)
printTaggedWarning("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
input.attrs.insert_or_assign(
"ref", chomp(TRY_AWAIT(runHg({"branch", "-R", actualUrl})))
);
auto files = tokenizeString<std::set<std::string>>(
runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
TRY_AWAIT(runHg(
{"status",
"-R",
actualUrl,
"--clean",
"--modified",
"--added",
"--no-status",
"--print0"}
)),
"\0"s
);
Path actualPath(absPath(actualUrl));
@@ -221,8 +237,9 @@ struct MercurialInputScheme : InputScheme
co_return {std::move(storePath), input};
}
auto tokens = tokenizeString<std::vector<std::string>>(
runHg({ "identify", "-R", actualUrl, "-r", ".", "--template", "{branch} {node}" }));
auto tokens = tokenizeString<std::vector<std::string>>(TRY_AWAIT(
runHg({"identify", "-R", actualUrl, "-r", ".", "--template", "{branch} {node}"})
));
assert(tokens.size() == 2);
input.attrs.insert_or_assign("ref", tokens[0]);
input.attrs.insert_or_assign("rev", tokens[1]);
@@ -291,26 +308,37 @@ struct MercurialInputScheme : InputScheme
if (pathExists(cacheDir)) {
try {
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
TRY_AWAIT(runHg({"pull", "-R", cacheDir, "--", actualUrl}));
} catch (ExecError & e) {
auto transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runHg({ "recover", "-R", cacheDir });
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
// cannot await in catch, rest of exception handled below
goto failed;
} else {
throw ExecError(e.status, "'hg pull' %s", statusToString(e.status));
}
}
if (false) {
failed:
/* hg throws "abandoned transaction" error only if this file exists */
TRY_AWAIT(runHg({"recover", "-R", cacheDir}));
TRY_AWAIT(runHg({"pull", "-R", cacheDir, "--", actualUrl}));
}
} else {
createDirs(dirOf(cacheDir));
runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir });
TRY_AWAIT(runHg({"clone", "--noupdate", "--", actualUrl, cacheDir}));
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
runHg({ "identify", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {count(revset('::{rev}'))} {branch}" }));
auto tokens = tokenizeString<std::vector<std::string>>(TRY_AWAIT(runHg(
{"identify",
"-R",
cacheDir,
"-r",
revOrRef,
"--template",
"{node} {count(revset('::{rev}'))} {branch}"}
)));
assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashType::SHA1).gitRev());
@@ -323,7 +351,9 @@ struct MercurialInputScheme : InputScheme
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runHg({ "archive", "-R", cacheDir, "-r", fmt("id(%s)", input.getRev()->gitRev()), tmpDir });
TRY_AWAIT(runHg(
{"archive", "-R", cacheDir, "-r", fmt("id(%s)", input.getRev()->gitRev()), tmpDir}
));
deletePath(tmpDir + "/.hg_archival.txt");