From 6567707dc1fe1ae93c761ec222172e8a077a4a9a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 16 May 2025 13:16:18 +0200 Subject: [PATCH] libstore: remove DrvOutputSubstitutionGoal this goal is only involved for output paths that aren't known at initial build time, which in turn can only happen if they are ca paths. since we can no longer create ca derivations during eval *or* read them from disk we can now assume that we will never run this goal. there are still some vestiges like output known-ness we can't remove yet, so those must stay. Change-Id: I989e5ad4600c628bcbe8e17e1b082ce8d73a3bd9 --- lix/libstore/build/derivation-goal.cc | 14 +- .../build/drv-output-substitution-goal.cc | 164 ------------------ .../build/drv-output-substitution-goal.hh | 80 --------- lix/libstore/build/worker.cc | 16 -- lix/libstore/build/worker.hh | 15 -- lix/libstore/meson.build | 2 - 6 files changed, 6 insertions(+), 285 deletions(-) delete mode 100644 lix/libstore/build/drv-output-substitution-goal.cc delete mode 100644 lix/libstore/build/drv-output-substitution-goal.hh diff --git a/lix/libstore/build/derivation-goal.cc b/lix/libstore/build/derivation-goal.cc index 91d1022bb..feb790b9d 100644 --- a/lix/libstore/build/derivation-goal.cc +++ b/lix/libstore/build/derivation-goal.cc @@ -10,7 +10,6 @@ #include "lix/libstore/common-protocol-impl.hh" // IWYU pragma: keep #include "lix/libstore/local-store.hh" // TODO remove, along with remaining downcasts #include "lix/libstore/build/substitution-goal.hh" -#include "lix/libstore/build/drv-output-substitution-goal.hh" #include "lix/libutil/result.hh" #include "lix/libutil/strings.hh" @@ -253,14 +252,13 @@ try { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; - if (!status.known) - dependencies.add( - worker.goalFactory().makeDrvOutputSubstitutionGoal( - DrvOutput{status.outputHash, outputName}, - buildMode == bmRepair ? Repair : NoRepair - ) + if (!status.known) { + // TODO remove somehow + throw Error( + "congrats, you hit vestigial CA code. sigh.\n" + "please report a bug at https://git.lix.systems/lix-project/lix/issues" ); - else { + } else { auto * cap = getDerivationCA(*drv); dependencies.add(worker.goalFactory().makePathSubstitutionGoal( status.known->path, diff --git a/lix/libstore/build/drv-output-substitution-goal.cc b/lix/libstore/build/drv-output-substitution-goal.cc deleted file mode 100644 index a5caa4464..000000000 --- a/lix/libstore/build/drv-output-substitution-goal.cc +++ /dev/null @@ -1,164 +0,0 @@ -#include "lix/libstore/build/drv-output-substitution-goal.hh" -#include "lix/libstore/build-result.hh" -#include "lix/libutil/async.hh" -#include "lix/libutil/finally.hh" -#include "lix/libstore/build/worker.hh" -#include "lix/libstore/build/substitution-goal.hh" -#include "lix/libutil/signals.hh" -#include -#include -#include - -namespace nix { - -DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( - const DrvOutput & id, - Worker & worker, - bool isDependency, - RepairFlag repair, - std::optional ca) - : Goal(worker, isDependency) - , id(id) -{ - name = fmt("substitution of '%s'", id.to_string()); - trace("created"); -} - - -kj::Promise> DrvOutputSubstitutionGoal::workImpl() noexcept -try { - trace("init"); - - /* If the derivation already exists, we’re done */ - if (TRY_AWAIT(worker.store.queryRealisation(id))) { - co_return WorkResult{ecSuccess}; - } - - subs = settings.useSubstitutes ? TRY_AWAIT(getDefaultSubstituters()) : std::list>(); - co_return co_await tryNext(); -} catch (...) { - co_return result::current_exception(); -} - -kj::Promise> DrvOutputSubstitutionGoal::tryNext() noexcept -try { - trace("trying next substituter"); - - if (!slotToken.valid()) { - slotToken = co_await worker.substitutions.acquire(); - } - - maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - - if (substituterFailed) { - worker.failedSubstitutions++; - } - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters}; - } - - sub = subs.front(); - subs.pop_front(); - - /* The async call to a curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - downloadState = std::make_shared(); - auto pipe = kj::newPromiseAndCrossThreadFulfiller(); - downloadState->outPipe = kj::mv(pipe.fulfiller); - - downloadState->result = - std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] { - Finally updateStats([&]() { downloadState->outPipe->fulfill(); }); - ReceiveInterrupts receiveInterrupts; - AsyncIoRoot aio; - return aio.blockOn(sub->queryRealisation(id)); - }); - - co_await pipe.promise; - co_return co_await realisationFetched(); -} catch (...) { - co_return result::current_exception(); -} - -kj::Promise> DrvOutputSubstitutionGoal::realisationFetched() noexcept -try { - maintainRunningSubstitutions.reset(); - slotToken = {}; - - try { - outputInfo = downloadState->result.get(); - } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) - printError(e.what()); - substituterFailed = true; - } - - if (!outputInfo) { - co_return co_await tryNext(); - } - - kj::Vector>>> dependencies; - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = TRY_AWAIT(worker.store.queryRealisation(depId)); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - co_return co_await tryNext(); - } - dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); - } - } - - dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - - if (!dependencies.empty()) { - TRY_AWAIT(waitForGoals(dependencies.releaseAsArray())); - } - co_return co_await outPathValid(); -} catch (...) { - co_return result::current_exception(); -} - -kj::Promise> DrvOutputSubstitutionGoal::outPathValid() noexcept -try { - assert(outputInfo); - trace("output path substituted"); - - if (nrFailed > 0) { - debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - co_return WorkResult{ - nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - }; - } - - TRY_AWAIT(worker.store.registerDrvOutput(*outputInfo)); - co_return TRY_AWAIT(finished()); -} catch (...) { - co_return result::current_exception(); -} - -kj::Promise> DrvOutputSubstitutionGoal::finished() noexcept -try { - trace("finished"); - return {WorkResult{ecSuccess}}; -} catch (...) { - return {result::current_exception()}; -} - -} diff --git a/lix/libstore/build/drv-output-substitution-goal.hh b/lix/libstore/build/drv-output-substitution-goal.hh deleted file mode 100644 index f82a68a27..000000000 --- a/lix/libstore/build/drv-output-substitution-goal.hh +++ /dev/null @@ -1,80 +0,0 @@ -#pragma once -///@file - -#include "lix/libutil/notifying-counter.hh" -#include "lix/libstore/store-api.hh" -#include "lix/libstore/build/goal.hh" -#include "lix/libstore/realisation.hh" -#include - -namespace nix { - -class Worker; - -/** - * Substitution of a derivation output. - * This is done in three steps: - * 1. Fetch the output info from a substituter - * 2. Substitute the corresponding output path - * 3. Register the output info - */ -class DrvOutputSubstitutionGoal : public Goal { - - /** - * The drv output we're trying to substitute - */ - DrvOutput id; - - /** - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - NotifyingCounter::Bump maintainRunningSubstitutions; - - struct DownloadState - { - kj::Own> outPipe; - std::future> result; - }; - - std::shared_ptr downloadState; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - -public: - DrvOutputSubstitutionGoal( - const DrvOutput & id, - Worker & worker, - bool isDependency, - RepairFlag repair = NoRepair, - std::optional ca = std::nullopt - ); - - kj::Promise> tryNext() noexcept; - kj::Promise> realisationFetched() noexcept; - kj::Promise> outPathValid() noexcept; - kj::Promise> finished() noexcept; - - kj::Promise> workImpl() noexcept override; - - JobCategory jobCategory() const override { - return JobCategory::Substitution; - }; -}; - -} diff --git a/lix/libstore/build/worker.cc b/lix/libstore/build/worker.cc index 4c6eadbb4..154fa461c 100644 --- a/lix/libstore/build/worker.cc +++ b/lix/libstore/build/worker.cc @@ -5,7 +5,6 @@ #include "lix/libstore/build/worker.hh" #include "lix/libutil/finally.hh" #include "lix/libstore/build/substitution-goal.hh" -#include "lix/libstore/build/drv-output-substitution-goal.hh" #include "lix/libstore/build/local-derivation-goal.hh" #include "lix/libutil/signals.hh" #include "lix/libstore/build/hook-instance.hh" // IWYU pragma: keep @@ -50,7 +49,6 @@ Worker::~Worker() children.clear(); derivationGoals.clear(); - drvOutputSubstitutionGoals.clear(); substitutionGoals.clear(); assert(expectedSubstitutions == 0); @@ -183,20 +181,6 @@ Worker::makePathSubstitutionGoal( } -std::pair, kj::Promise>> -Worker::makeDrvOutputSubstitutionGoal( - const DrvOutput & id, RepairFlag repair, std::optional ca -) -{ - return makeGoalCommon( - drvOutputSubstitutionGoals, - id, - [&] { return std::make_unique(id, *this, running, repair, ca); }, - [&](auto &) { return true; } - ); -} - - std::pair>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { diff --git a/lix/libstore/build/worker.hh b/lix/libstore/build/worker.hh index 7fc3d1fe9..ea6fd0f40 100644 --- a/lix/libstore/build/worker.hh +++ b/lix/libstore/build/worker.hh @@ -21,7 +21,6 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; struct PathSubstitutionGoal; -class DrvOutputSubstitutionGoal; class LocalStore; typedef std::chrono::time_point steady_time_point; @@ -54,12 +53,6 @@ public: RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) = 0; - virtual std::pair, kj::Promise>> - makeDrvOutputSubstitutionGoal( - const DrvOutput & id, - RepairFlag repair = NoRepair, - std::optional ca = std::nullopt - ) = 0; /** * Make a goal corresponding to the `DerivedPath`. @@ -76,7 +69,6 @@ class WorkerBase : protected GoalFactory { friend struct DerivationGoal; friend struct PathSubstitutionGoal; - friend class DrvOutputSubstitutionGoal; protected: GoalFactory & goalFactory() { return *this; } @@ -139,7 +131,6 @@ private: */ std::map> derivationGoals; std::map> substitutionGoals; - std::map> drvOutputSubstitutionGoals; /** * Cache for pathContentsGood(). @@ -265,12 +256,6 @@ private: RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) override; - std::pair, kj::Promise>> - makeDrvOutputSubstitutionGoal( - const DrvOutput & id, - RepairFlag repair = NoRepair, - std::optional ca = std::nullopt - ) override; /** * Make a goal corresponding to the `DerivedPath`. diff --git a/lix/libstore/meson.build b/lix/libstore/meson.build index aaa39e266..1dd83c477 100644 --- a/lix/libstore/meson.build +++ b/lix/libstore/meson.build @@ -140,7 +140,6 @@ libstore_sources = files( 'build-result.cc', 'build/child.cc', 'build/derivation-goal.cc', - 'build/drv-output-substitution-goal.cc', 'build/entry-points.cc', 'build/goal.cc', 'build/hook-instance.cc', @@ -209,7 +208,6 @@ libstore_headers = files( 'build-result.hh', 'build/child.hh', 'build/derivation-goal.hh', - 'build/drv-output-substitution-goal.hh', 'build/goal.hh', 'build/hook-instance.hh', 'build/local-derivation-goal.hh',