guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify FSAccessor, listNar

Browse Source 
Change-Id: I804ac18c4b6b80699247bb885493056799bf3d0c
This commit is contained in:
eldritch horrors
2025-03-05 18:49:45 +01:00
parent 19b8502ced
commit a29e3a4a58
11 changed files with 138 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/libstore/fs-accessor.hh
+7 -4
View File
@@ -1,7 +1,9 @@
#pragma once
///@file
#include "lix/libutil/result.hh"
#include "lix/libutil/types.hh"
#include <kj/async.h>
namespace nix {
@@ -33,9 +35,9 @@ public:
virtual ~FSAccessor() { }
virtual Stat stat(const Path & path) = 0;
virtual kj::Promise<Result<Stat>> stat(const Path & path) = 0;
virtual StringSet readDirectory(const Path & path) = 0;
virtual kj::Promise<Result<StringSet>> readDirectory(const Path & path) = 0;
/**
* Read a file inside the store.
@@ -44,9 +46,10 @@ public:
* inside a valid store path, otherwise it just needs to be physically
* present (but not necessarily properly registered)
*/
virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
virtual kj::Promise<Result<std::string>>
readFile(const Path & path, bool requireValidPath = true) = 0;
virtual std::string readLink(const Path & path) = 0;
virtual kj::Promise<Result<std::string>> readLink(const Path & path) = 0;
};
}
lix/libstore/local-fs-store.cc
+29 -18
View File
@@ -14,38 +14,42 @@ struct LocalStoreAccessor : public FSAccessor
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
Path toRealPath(const Path & path, bool requireValidPath = true)
{
kj::Promise<Result<Path>> toRealPath(const Path & path, bool requireValidPath = true)
try {
auto storePath = store->toStorePath(path).first;
if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' does not exist in the store", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->config().storeDir.size());
co_return store->getRealStoreDir() + std::string(path, store->config().storeDir.size());
} catch (...) {
co_return result::current_exception();
}
FSAccessor::Stat stat(const Path & path) override
{
auto realPath = toRealPath(path);
kj::Promise<Result<FSAccessor::Stat>> stat(const Path & path) override
try {
auto realPath = TRY_AWAIT(toRealPath(path));
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
if (errno == ENOENT || errno == ENOTDIR) co_return {Type::tMissing, 0, false};
throw SysError("getting status of '%1%'", path);
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error("file '%1%' has unsupported type", path);
return {
co_return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
} catch (...) {
co_return result::current_exception();
}
StringSet readDirectory(const Path & path) override
{
auto realPath = toRealPath(path);
kj::Promise<Result<StringSet>> readDirectory(const Path & path) override
try {
auto realPath = TRY_AWAIT(toRealPath(path));
auto entries = nix::readDirectory(realPath);
@@ -53,17 +57,24 @@ struct LocalStoreAccessor : public FSAccessor
for (auto & entry : entries)
res.insert(entry.name);
return res;
co_return res;
} catch (...) {
co_return result::current_exception();
}
std::string readFile(const Path & path, bool requireValidPath = true) override
{
return nix::readFile(toRealPath(path, requireValidPath));
kj::Promise<Result<std::string>>
readFile(const Path & path, bool requireValidPath = true) override
try {
co_return nix::readFile(TRY_AWAIT(toRealPath(path, requireValidPath)));
} catch (...) {
co_return result::current_exception();
}
std::string readLink(const Path & path) override
{
return nix::readLink(toRealPath(path));
kj::Promise<Result<std::string>> readLink(const Path & path) override
try {
co_return nix::readLink(TRY_AWAIT(toRealPath(path)));
} catch (...) {
co_return result::current_exception();
}
};
lix/libstore/nar-accessor.cc
+33 -21
View File
@@ -1,5 +1,6 @@
#include "lix/libstore/nar-accessor.hh"
#include "lix/libutil/archive.hh"
#include "lix/libutil/async.hh"
#include <map>
#include <memory>
@@ -87,11 +88,11 @@ struct NarAccessor : public FSAccessor
return *result;
}
Stat stat(const Path & path) override
{
kj::Promise<Result<Stat>> stat(const Path & path) override
try {
auto i = find(path);
if (i == nullptr)
return {FSAccessor::Type::tMissing, 0, false};
co_return {FSAccessor::Type::tMissing, 0, false};
auto handlers = overloaded{
[](const nar_index::File & f) {
return Stat{tRegular, f.size, f.executable, f.offset};
@@ -99,11 +100,13 @@ struct NarAccessor : public FSAccessor
[](const nar_index::Symlink &) { return Stat{tSymlink}; },
[](const nar_index::Directory &) { return Stat{tDirectory}; },
};
return std::visit(handlers, *i);
co_return std::visit(handlers, *i);
} catch (...) {
co_return result::current_exception();
}
StringSet readDirectory(const Path & path) override
{
kj::Promise<Result<StringSet>> readDirectory(const Path & path) override
try {
auto & i = get(path);
auto dir = std::get_if<nar_index::Directory>(&i);
@@ -114,29 +117,36 @@ struct NarAccessor : public FSAccessor
for (auto & child : dir->contents)
res.insert(child.first);
return res;
co_return res;
} catch (...) {
co_return result::current_exception();
}
std::string readFile(const Path & path, bool requireValidPath = true) override
{
kj::Promise<Result<std::string>>
readFile(const Path & path, bool requireValidPath = true) override
try {
auto & i = get(path);
auto file = std::get_if<nar_index::File>(&i);
if (!file)
throw Error("path '%1%' inside NAR file is not a regular file", path);
if (getNarBytes) return getNarBytes(file->offset, file->size);
if (getNarBytes) co_return getNarBytes(file->offset, file->size);
assert(nar);
return std::string(*nar, file->offset, file->size);
co_return std::string(*nar, file->offset, file->size);
} catch (...) {
co_return result::current_exception();
}
std::string readLink(const Path & path) override
{
kj::Promise<Result<std::string>> readLink(const Path & path) override
try {
auto & i = get(path);
auto link = std::get_if<nar_index::Symlink>(&i);
if (!link)
throw Error("path '%1%' inside NAR file is not a symlink", path);
return link->target;
co_return link->target;
} catch (...) {
co_return result::current_exception();
}
};
@@ -157,9 +167,9 @@ ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
}
using nlohmann::json;
json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
{
auto st = accessor->stat(path);
kj::Promise<Result<json>> listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
try {
auto st = TRY_AWAIT(accessor->stat(path));
json obj = json::object();
@@ -177,9 +187,9 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
{
obj["entries"] = json::object();
json &res2 = obj["entries"];
for (auto & name : accessor->readDirectory(path)) {
for (auto & name : TRY_AWAIT(accessor->readDirectory(path))) {
if (recurse) {
res2[name] = listNar(accessor, path + "/" + name, true);
res2[name] = TRY_AWAIT(listNar(accessor, path + "/" + name, true));
} else
res2[name] = json::object();
}
@@ -187,13 +197,15 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
break;
case FSAccessor::Type::tSymlink:
obj["type"] = "symlink";
obj["target"] = accessor->readLink(path);
obj["target"] = TRY_AWAIT(accessor->readLink(path));
break;
case FSAccessor::Type::tMissing:
default:
throw Error("path '%s' does not exist in NAR", path);
}
return obj;
co_return obj;
} catch (...) {
co_return result::current_exception();
}
static nlohmann::json listNar(const nar_index::Entry & e, Path path)
lix/libstore/nar-accessor.hh
+2 -1
View File
@@ -37,7 +37,8 @@ ref<FSAccessor> makeLazyNarAccessor(
* Write a JSON representation of the contents of a NAR (except file
* contents).
*/
nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
kj::Promise<Result<nlohmann::json>>
listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
nlohmann::json listNar(const nar_index::Entry & nar);
}
lix/libstore/remote-fs-accessor.cc
+41 -26
View File
@@ -22,8 +22,9 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
{
kj::Promise<Result<ref<FSAccessor>>>
RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
try {
if (cacheDir != "") {
try {
/* FIXME: do this asynchronously. */
@@ -38,18 +39,21 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
if (cacheDir != "") {
try {
nlohmann::json j = listNar(narAccessor, "", true);
nlohmann::json j = TRY_AWAIT(listNar(narAccessor, "", true));
writeFile(makeCacheFile(hashPart, "ls"), j.dump());
} catch (...) {
ignoreExceptionExceptInterrupt();
}
}
return narAccessor;
co_return narAccessor;
} catch (...) {
co_return result::current_exception();
}
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
{
kj::Promise<Result<std::pair<ref<FSAccessor>, Path>>>
RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
try {
auto path = canonPath(path_);
auto [storePath, restPath] = store->toStorePath(path);
@@ -58,7 +62,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
throw InvalidPath("path '%1%' does not exist in remote store", store->printStorePath(storePath));
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
if (i != nars.end()) co_return {i->second, restPath};
std::string listing;
Path cacheFile;
@@ -85,44 +89,55 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
});
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
co_return {narAccessor, restPath};
} catch (SysError &) { }
try {
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
co_return {narAccessor, restPath};
} catch (SysError &) { }
}
StringSink sink;
store->narFromPath(storePath)->drainInto(sink);
return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
co_return {TRY_AWAIT(addToCache(storePath.hashPart(), std::move(sink.s))), restPath};
} catch (...) {
co_return result::current_exception();
}
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
{
auto res = fetch(path);
return res.first->stat(res.second);
kj::Promise<Result<FSAccessor::Stat>> RemoteFSAccessor::stat(const Path & path)
try {
auto res = TRY_AWAIT(fetch(path));
co_return TRY_AWAIT(res.first->stat(res.second));
} catch (...) {
co_return result::current_exception();
}
StringSet RemoteFSAccessor::readDirectory(const Path & path)
{
auto res = fetch(path);
return res.first->readDirectory(res.second);
kj::Promise<Result<StringSet>> RemoteFSAccessor::readDirectory(const Path & path)
try {
auto res = TRY_AWAIT(fetch(path));
co_return TRY_AWAIT(res.first->readDirectory(res.second));
} catch (...) {
co_return result::current_exception();
}
std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
{
auto res = fetch(path, requireValidPath);
return res.first->readFile(res.second);
kj::Promise<Result<std::string>>
RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
try {
auto res = TRY_AWAIT(fetch(path, requireValidPath));
co_return TRY_AWAIT(res.first->readFile(res.second));
} catch (...) {
co_return result::current_exception();
}
std::string RemoteFSAccessor::readLink(const Path & path)
{
auto res = fetch(path);
return res.first->readLink(res.second);
kj::Promise<Result<std::string>> RemoteFSAccessor::readLink(const Path & path)
try {
auto res = TRY_AWAIT(fetch(path));
co_return TRY_AWAIT(res.first->readLink(res.second));
} catch (...) {
co_return result::current_exception();
}
}
lix/libstore/remote-fs-accessor.hh
+8 -6
View File
@@ -15,26 +15,28 @@ class RemoteFSAccessor : public FSAccessor
Path cacheDir;
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true);
kj::Promise<Result<std::pair<ref<FSAccessor>, Path>>>
fetch(const Path & path_, bool requireValidPath = true);
friend class BinaryCacheStore;
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar);
kj::Promise<Result<ref<FSAccessor>>> addToCache(std::string_view hashPart, std::string && nar);
public:
RemoteFSAccessor(ref<Store> store,
const /* FIXME: use std::optional */ Path & cacheDir = "");
Stat stat(const Path & path) override;
kj::Promise<Result<Stat>> stat(const Path & path) override;
StringSet readDirectory(const Path & path) override;
kj::Promise<Result<StringSet>> readDirectory(const Path & path) override;
std::string readFile(const Path & path, bool requireValidPath = true) override;
kj::Promise<Result<std::string>>
readFile(const Path & path, bool requireValidPath = true) override;
std::string readLink(const Path & path) override;
kj::Promise<Result<std::string>> readLink(const Path & path) override;
};
}
lix/libstore/store-api.cc
+5 -3
View File
@@ -1392,9 +1392,11 @@ readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPa
try {
auto accessor = store.getFSAccessor();
try {
co_return parseDerivation(store,
accessor->readFile(store.printStorePath(drvPath), requireValidPath),
Derivation::nameFromPath(drvPath));
co_return parseDerivation(
store,
TRY_AWAIT(accessor->readFile(store.printStorePath(drvPath), requireValidPath)),
Derivation::nameFromPath(drvPath)
);
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
}
lix/nix/cat.cc
+2 -2
View File
@@ -11,13 +11,13 @@ struct MixCat : virtual Args
void cat(ref<FSAccessor> accessor)
{
auto st = accessor->stat(path);
auto st = aio().blockOn(accessor->stat(path));
if (st.type == FSAccessor::Type::tMissing)
throw Error("path '%1%' does not exist", path);
if (st.type != FSAccessor::Type::tRegular)
throw Error("path '%1%' is not a regular file", path);
auto file = accessor->readFile(path);
auto file = aio().blockOn(accessor->readFile(path));
logger->pause();
writeFull(STDOUT_FILENO, file);
lix/nix/ls.cc
+6 -6
View File
@@ -45,7 +45,7 @@ struct MixLs : virtual Args, MixJSON
auto showFile = [&](const Path & curPath, const std::string & relPath) {
if (verbose) {
auto st = accessor->stat(curPath);
auto st = aio().blockOn(accessor->stat(curPath));
std::string tp =
st.type == FSAccessor::Type::tRegular ?
(st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
@@ -53,14 +53,14 @@ struct MixLs : virtual Args, MixJSON
"dr-xr-xr-x";
auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
if (st.type == FSAccessor::Type::tSymlink)
line += " -> " + accessor->readLink(curPath);
line += " -> " + aio().blockOn(accessor->readLink(curPath));
logger->cout(line);
if (recursive && st.type == FSAccessor::Type::tDirectory)
doPath(st, curPath, relPath, false);
} else {
logger->cout(relPath);
if (recursive) {
auto st = accessor->stat(curPath);
auto st = aio().blockOn(accessor->stat(curPath));
if (st.type == FSAccessor::Type::tDirectory)
doPath(st, curPath, relPath, false);
}
@@ -71,14 +71,14 @@ struct MixLs : virtual Args, MixJSON
const std::string & relPath, bool showDirectory)
{
if (st.type == FSAccessor::Type::tDirectory && !showDirectory) {
auto names = accessor->readDirectory(curPath);
auto names = aio().blockOn(accessor->readDirectory(curPath));
for (auto & name : names)
showFile(curPath + "/" + name, relPath + "/" + name);
} else
showFile(curPath, relPath);
};
auto st = accessor->stat(path);
auto st = aio().blockOn(accessor->stat(path));
if (st.type == FSAccessor::Type::tMissing)
throw Error("path '%1%' does not exist", path);
doPath(st, path,
@@ -93,7 +93,7 @@ struct MixLs : virtual Args, MixJSON
if (json) {
if (showDirectory)
throw UsageError("'--directory' is useless with '--json'");
logger->cout("%s", listNar(accessor, path, recursive));
logger->cout("%s", aio().blockOn(listNar(accessor, path, recursive)));
} else
listText(accessor);
}
lix/nix/run.cc
+1 -1
View File
@@ -139,7 +139,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
pathAdditions.push_back(store->printStorePath(path) + "/bin");
auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages";
if (accessor->stat(propPath).type == FSAccessor::tRegular) {
if (aio().blockOn(accessor->stat(propPath)).type == FSAccessor::tRegular) {
for (auto & p : tokenizeString<Paths>(readFile(propPath)))
todo.push(store->parseStorePath(p));
}
lix/nix/why-depends.cc
+4 -4
View File
@@ -213,7 +213,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
std::function<void(const Path &)> visitPath;
visitPath = [&](const Path & p) {
auto st = accessor->stat(p);
auto st = aio().blockOn(accessor->stat(p));
auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1);
@@ -222,13 +222,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
};
if (st.type == FSAccessor::Type::tDirectory) {
auto names = accessor->readDirectory(p);
auto names = aio().blockOn(accessor->readDirectory(p));
for (auto & name : names)
visitPath(p + "/" + name);
}
else if (st.type == FSAccessor::Type::tRegular) {
auto contents = accessor->readFile(p);
auto contents = aio().blockOn(accessor->readFile(p));
for (auto & hash : hashes) {
auto pos = contents.find(hash);
@@ -246,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
}
else if (st.type == FSAccessor::Type::tSymlink) {
auto target = accessor->readLink(p);
auto target = aio().blockOn(accessor->readLink(p));
for (auto & hash : hashes) {
auto pos = target.find(hash);