From 29b07611b8aabc68a56aee1be4e327da9e8296ee Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 8 Mar 2025 16:39:57 +0000 Subject: [PATCH] gc: allow continuing deletion even if some paths are still live This adds a gcTryDeleteSpecific operation. This is similar to gcDeleteSpecific, but will not fail if any of the given paths cannot be deleted. Paths that could not be deleted are reported in the new `kept` field of struct GCResults. This also changes the behaviour of gcDeleteSpecific, such that it will now continue deleting paths even if it fails to delete one along the way, and only throw an error once deletion of all the given paths has been attempted. This seems reasonable to me, because it makes its behaviour somewhat less surprising -- previously, if some paths were deletable and others weren't, the deletable ones would be deleted iff they preceded the live ones in lexical sort order. This also fixes a regression introduced in 8614cf13344eca75074cd4af20fd90238571b0b6, whereby nix-store --delete failed to delete paths if they had any dependents -- even if none of the dependents had GC roots. The gcTryDeleteSpecific operation is surfaced via additional flags for the `nix store delete` and `nix-store --delete` commands. This makes custom garbage-collection logic a lot easier to implement and experiment with: - Paths known to be large can be thrown at `nix store delete` without having to manually filter out those that are still reachable from a root, e.g. `nix store delete /nix/store/*mbrola-voices*` - The --delete-closure option allows extending this to paths that are not large themselves but do have a large closure size, e.g. `nix store delete /nix/store/*nixos-system-gamingpc*` Having an option for this is not strictly necessary, but convenient because it doesn't require the user to add an extra `nix-store -qR` (or `nix path-info -r`) into their command, nor to rewrite their command to use `--stdin` if the closure ends up too large to fit on a command line. - Other heuristics like atime-based deletion can be applied more easily, because `nix store delete` once again takes over the task of working out which paths can't be deleted. Change-Id: If345407fe7b11bdb3a8fdc04b0d56c32ab3d5928 --- doc/manual/rl-next/delete-no-early-fail.md | 18 +++++++ doc/manual/rl-next/delete-skip-live.md | 26 ++++++++++ .../src/command-ref/nix-store/delete.md | 9 +++- lix/legacy/nix-store.cc | 13 ++++- lix/libstore/gc-store.hh | 12 +++++ lix/libstore/gc.cc | 49 ++++++++++++++----- lix/nix/store-delete.cc | 20 +++++++- tests/functional/gc.sh | 19 +++++++ 8 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 doc/manual/rl-next/delete-no-early-fail.md create mode 100644 doc/manual/rl-next/delete-skip-live.md diff --git a/doc/manual/rl-next/delete-no-early-fail.md b/doc/manual/rl-next/delete-no-early-fail.md new file mode 100644 index 000000000..0c6cbe12d --- /dev/null +++ b/doc/manual/rl-next/delete-no-early-fail.md @@ -0,0 +1,18 @@ +--- +synopsis: 'Deletion of specific paths no longer fails fast' +issues: [] +cls: [2778] +category: Improvements +credits: [lheckemann] +--- + +`nix-store --delete` and `nix store delete` now continue deleting +paths even if some of the given paths are still live. An error is only +thrown once deletion of all the given paths has been +attempted. Previously, if some paths were deletable and others +weren't, the deletable ones would be deleted iff they preceded the +live ones in lexical sort order. + +The error message for still-live paths no longer reports the paths +that could not be deleted, because there could potentially be many of +these. diff --git a/doc/manual/rl-next/delete-skip-live.md b/doc/manual/rl-next/delete-skip-live.md new file mode 100644 index 000000000..ed013baff --- /dev/null +++ b/doc/manual/rl-next/delete-skip-live.md @@ -0,0 +1,26 @@ +--- +synopsis: '`--skip-live` for path deletion' +issues: [] +cls: [2778] +category: Improvements +credits: [lheckemann] +--- + +`nix-store --delete` and `nix store delete` now support a +`--skip-live` option and a `--delete-closure` option. + +This makes custom garbage-collection logic a lot easier to implement +and experiment with: + +- Paths known to be large can be thrown at `nix store delete` without + having to manually filter out those that are still reachable from a + root, e.g. + `nix store delete /nix/store/*mbrola-voices*` + +- The `--delete-closure` option allows extending this to paths that are + not large themselves but do have a large closure size, e.g. + `nix store delete /nix/store/*nixos-system-gamingpc*`. + +- Other heuristics like atime-based deletion can be applied more + easily, because `nix store delete` once again takes over the task of + working out which paths can't be deleted. diff --git a/doc/manual/src/command-ref/nix-store/delete.md b/doc/manual/src/command-ref/nix-store/delete.md index 550c5ea29..3edd9a3c4 100644 --- a/doc/manual/src/command-ref/nix-store/delete.md +++ b/doc/manual/src/command-ref/nix-store/delete.md @@ -4,7 +4,7 @@ # Synopsis -`nix-store` `--delete` [`--ignore-liveness`] *paths…* +`nix-store` `--delete` [`--ignore-liveness`] [`--skip-live`] [`--delete-closure`] *paths…* # Description @@ -18,6 +18,13 @@ With the option `--ignore-liveness`, reachability from the roots is ignored. However, the path still won’t be deleted if there are other paths in the store that refer to it (i.e., depend on it). +This operation will raise an error if any of the paths are still live +and `--ignore-liveness` is not passed. Passing `--skip-live` will +prevent this from being considered an error. + +The option `--delete-closure` will also attempt to delete any paths +that are in the given path's dependency closure. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index 7d6b8c181..7eab7c69a 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -677,13 +677,22 @@ static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) { GCOptions options; options.action = GCOptions::gcDeleteSpecific; + bool deleteClosure = false; for (auto & i : opFlags) if (i == "--ignore-liveness") options.ignoreLiveness = true; + else if (i == "--skip-live") options.action = GCOptions::gcTryDeleteSpecific; + else if (i == "--delete-closure") deleteClosure = true; else throw UsageError("unknown flag '%1%'", i); - for (auto & i : opArgs) - options.pathsToDelete.insert(store->followLinksToStorePath(i)); + for (auto & arg : opArgs) { + StorePath path = store->followLinksToStorePath(arg); + if (deleteClosure) { + aio.blockOn(store->computeFSClosure(path, options.pathsToDelete)); + } else { + options.pathsToDelete.insert(path); + } + } auto & gcStore = require(*store); diff --git a/lix/libstore/gc-store.hh b/lix/libstore/gc-store.hh index 94b2480c9..121c00ceb 100644 --- a/lix/libstore/gc-store.hh +++ b/lix/libstore/gc-store.hh @@ -32,13 +32,19 @@ struct GCOptions * - `gcDeleteDead`: actually delete the latter set. * * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, failing if any are still reachable. + * + * - `gcTryDeleteSpecific`: delete the paths listed in * `pathsToDelete`, insofar as they are not reachable. + * Any that could not be deleted are returned via the + * `kept` field of GCResults. */ typedef enum { gcReturnLive, gcReturnDead, gcDeleteDead, gcDeleteSpecific, + gcTryDeleteSpecific, } GCAction; GCAction action{gcDeleteDead}; @@ -71,6 +77,12 @@ struct GCResults */ PathSet paths; + /** + * If the action was gcTryDeleteSpecific, the paths that were not + * deleted because they are still live. + */ + PathSet kept; + /** * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the * number of bytes that would be or was freed. diff --git a/lix/libstore/gc.cc b/lix/libstore/gc.cc index 5e9c14bb3..cdf0204a3 100644 --- a/lix/libstore/gc.cc +++ b/lix/libstore/gc.cc @@ -597,7 +597,8 @@ GCOperation::~GCOperation() kj::Promise> LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { - bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool deleteSpecific = options.action == GCOptions::gcDeleteSpecific || options.action == GCOptions::gcTryDeleteSpecific; + bool shouldDelete = options.action == GCOptions::gcDeleteDead || deleteSpecific; bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; @@ -607,7 +608,7 @@ try { consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + if (deleteSpecific && options.ignoreLiveness) { gcKeepOutputs = false; gcKeepDerivations = false; } @@ -812,17 +813,41 @@ try { }; /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { - + paths (for gcDeleteSpecific and gcTryDeleteSpecific). */ + if (deleteSpecific) { + PathSet kept; for (auto & i : options.pathsToDelete) { TRY_AWAIT(deleteReferrersClosure(i)); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); + if (!dead.count(i)) { + std::string path(i.to_string()); + kept.insert(path); + results.kept.insert(path); + } + } + if (!kept.empty()) { + printTalkative("Paths not deleted because they are still referenced by GC roots:"); + for (auto &path: kept) { + printTalkative(path); + } + } + if (options.action == GCOptions::gcDeleteSpecific && !kept.empty()) { + std::ostringstream pathSummary; + for (auto const [n, path]: enumerate(kept)) { + pathSummary << "\n " << path; + const int summaryThreshold = 10; + if (n >= summaryThreshold) { + pathSummary << "\nand " << kept.size() - summaryThreshold << " others.\n"; + break; + } + } + throw Error( + "Cannot delete some of the given paths because they are still alive. " + "Paths not deleted:" + "%1%" + "To find out why, use nix-store --query --roots and nix-store --query --referrers." + , + pathSummary.str() + ); } } else if (options.maxFreed > 0) { @@ -874,7 +899,7 @@ try { safely deleted. FIXME: race condition with optimisePath(): we might see a link count of 1 just before optimisePath() increases the link count. */ - if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { + if (options.action == GCOptions::gcDeleteDead || deleteSpecific) { printInfo("deleting unused links..."); AutoCloseDir dir(opendir(linksDir.c_str())); diff --git a/lix/nix/store-delete.cc b/lix/nix/store-delete.cc index cef62896f..4a4b59e3f 100644 --- a/lix/nix/store-delete.cc +++ b/lix/nix/store-delete.cc @@ -10,6 +10,7 @@ using namespace nix; struct CmdStoreDelete : StorePathsCommand { GCOptions options { .action = GCOptions::gcDeleteSpecific }; + bool deleteClosure = false; CmdStoreDelete() { @@ -18,6 +19,16 @@ struct CmdStoreDelete : StorePathsCommand .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true} }); + addFlag({ + .longName = "skip-live", + .description = "Skip deleting any paths that are reachable from a root.", + .handler = {&options.action, GCOptions::gcTryDeleteSpecific} + }); + addFlag({ + .longName = "delete-closure", + .description = "Also attempt to delete all paths in the given paths' closures.", + .handler = {&deleteClosure, true} + }); realiseMode = Realise::Nothing; } @@ -37,8 +48,13 @@ struct CmdStoreDelete : StorePathsCommand { auto & gcStore = require(*store); - for (auto & path : storePaths) - options.pathsToDelete.insert(path); + for (auto & path : storePaths) { + if (deleteClosure) { + aio().blockOn(store->computeFSClosure(path, options.pathsToDelete)); + } else { + options.pathsToDelete.insert(path); + } + } GCResults results; PrintFreed freed(true, results); diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index 4923dc069..7e3327f4f 100644 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -58,3 +58,22 @@ nix-collect-garbage # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR + +createAndRootPaths +input0=$(< $outPath/reference-to-input-2/input0) +input2=$(readLink $outPath/reference-to-input-2) +fodDrv=$(nix-store -qR $drvPath | grep fod-input.drv) +fodOut=$(nix-store -q --outputs $fodDrv) +# path is still live but command should still succeed because of --skip-live +nix-store --delete --skip-live $(readLink $outPath/reference-to-input-2) + +rm "$NIX_STATE_DIR"/gcroots/foo +# with the dependent unrooted, we should be able to remove input2... +nix-store --delete --delete-closure $input2 +# which should remove input0, since only input2 and top depended on it and we passed --delete-closure +! test -e $input0 +# but fod should be unaffected, since it's not part of input-2's closure +test -e $fodOut + + +nix-collect-garbage