libstore: optimize nar copy stream adapter
we don't need to report progress for every read call. that's way too much. batching like this greatly reduces CPU usage for copies out of or into remote buidlers due to likewise greatly reduced log traffic. Change-Id: I3db2b2ab113fbaadefc69cfde6f977fb0c6cd5ad
This commit is contained in:
@@ -1031,8 +1031,17 @@ struct CopyPathStream : AsyncInputStream
|
||||
kj::Promise<Result<std::optional<size_t>>> read(void * data, size_t len) override
|
||||
try {
|
||||
auto result = TRY_AWAIT(inner->read(data, len));
|
||||
|
||||
// do not log progress on every call. nar copies cause a lot of small
|
||||
// reads, letting each read report the current copy progress causes a
|
||||
// huge amount of overhead (20x or more) in log traffic. reporting at
|
||||
// 64 kiB intervals is probably enough, being about 1000 dir entries.
|
||||
constexpr size_t CHUNK = 65536;
|
||||
const auto doLog = !result || copied / CHUNK < (copied + *result) / CHUNK || *result < len;
|
||||
if (result) {
|
||||
copied += *result;
|
||||
}
|
||||
if (doLog) {
|
||||
act.progress(copied, expected);
|
||||
}
|
||||
co_return result;
|
||||
|
||||
Reference in New Issue
Block a user