From 6d6cccee75ad63abc5e200ffd93575c92fdee2c0 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 18 Jul 2025 13:43:02 +0200 Subject: [PATCH] libstore: restrict build-hook parallelism previously we only had one build hook in waiting at most because build hook rpc was synchronous. now that it no longer is we attempt to start one hook per derivation, which depending on scheduling can be a *very* large number. restrict the waiting hook count to 4 to some concurrency without collecting a large number of hooks that may never do anything. Change-Id: Ic0b1125cec4acd69e8a0d4639c232e71b825e01d --- lix/libstore/build/derivation-goal.cc | 6 ++++++ lix/libstore/build/worker.hh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/lix/libstore/build/derivation-goal.cc b/lix/libstore/build/derivation-goal.cc index 7eb51a027..fc8640442 100644 --- a/lix/libstore/build/derivation-goal.cc +++ b/lix/libstore/build/derivation-goal.cc @@ -1029,6 +1029,9 @@ try { co_return HookResult::Decline{}; } + // make sure we don't launch an unbounded number of build hooks + auto hookSlot = co_await worker.hook.instancesSem.acquire(); + if (!worker.hook.instances.empty()) { hook = std::move(worker.hook.instances.front()); worker.hook.instances.pop_front(); @@ -1062,6 +1065,9 @@ try { throw Error("bad hook reply '%s'", buildResp.which()); } + // the build was accepted by the hook, we can free the slot for another build now + hookSlot = {}; + machineName = rpc::to(buildResp.getAccept().getMachineName()); auto runReq = buildResp.getAccept().getMachine().runRequest(); diff --git a/lix/libstore/build/worker.hh b/lix/libstore/build/worker.hh index ebdd50965..096f99316 100644 --- a/lix/libstore/build/worker.hh +++ b/lix/libstore/build/worker.hh @@ -204,6 +204,10 @@ private: public: struct HookState { + // make sure we never have too many remote build hook processes waiting. they are + // expensive to start and may be requested in great numbers for wide build trees. + // the original implementation only allowed a single process, we allow some more. + AsyncSemaphore instancesSem{4}; std::list> instances; /**