libstore: recursively delete cgroups

fixes #1143

Change-Id: I9f5cf5c127ac23e60e3232d85f338e467ed4f170

lix/libutil/cgroup.cc

@@ -2,7 +2,9 @@
 #include "error.hh"
 #include "file-descriptor.hh"
 #include "logging.hh"
+#include <deque>
 #include <fcntl.h>
+#include <filesystem>
 #include <sys/poll.h>
 #if __linux__
 
@@ -170,8 +172,29 @@ destroyCgroup(const std::string & name, const std::filesystem::path & aliveCgrou
 
     Result<CgroupStats> stats = readStatistics(aliveCgroup);
 
-    if (sys::rmdir(aliveCgroup) == -1) {
-        throw SysError("deleting cgroup '%s' at '%s'", name, aliveCgroup);
+    std::deque<std::filesystem::path> pending, toDelete;
+
+    pending.push_back(aliveCgroup);
+    while (!pending.empty()) {
+        auto current = std::move(pending.back());
+        pending.pop_back();
+
+        toDelete.push_front(current);
+        try {
+            for (auto sub : std::filesystem::directory_iterator(current)) {
+                if (sub.is_directory()) {
+                    pending.push_front(sub);
+                }
+            }
+        } catch (std::filesystem::filesystem_error & e) { // NOLINT(lix-foreign-exceptions)
+            throw Error("listing cgroup '%s': %s", name, e.what());
+        }
+    }
+
+    for (auto & candidate : toDelete) {
+        if (sys::rmdir(candidate) == -1) {
+            throw SysError("deleting cgroup '%s' at '%s'", name, aliveCgroup);
+        }
     }
 
     debug("cgroup '%s' destroyed", name);

tests/nixos/cgroups/default.nix

@@ -1,5 +1,18 @@
-{ nixpkgs, ... }:
+{ nixpkgs, pkgs, ... }:
 
+let
+  nestedCgroupsExpr = config: pkgs.writeText "nested.nix" ''
+    let utils = builtins.storePath ${config.system.build.extraUtils}; in
+    derivation {
+      name = "nested-cgroups";
+      system = builtins.currentSystem;
+      requiredSystemFeatures = [ "uid-range" ];
+      PATH = "''${utils}/bin";
+      builder = "''${utils}/bin/sh";
+      args = [ "-c" "mount -t cgroup2 none /sys/fs/cgroup; mkdir -p /sys/fs/cgroup/demo; touch $out" ];
+    }
+  '';
+in
 {
   name = "cgroups";
 
@@ -7,10 +20,12 @@
     {
       host =
         { config, pkgs, ... }:
-        { virtualisation.additionalPaths = [ pkgs.stdenvNoCC ];
+        { virtualisation.additionalPaths = [ pkgs.stdenvNoCC config.system.build.extraUtils ];
+          virtualisation.writableStore = true;
           nix.extraOptions =
             ''
               extra-experimental-features = nix-command auto-allocate-uids cgroups
+              auto-allocate-uids = true
               extra-system-features = uid-range
             '';
           nix.settings = {
@@ -46,6 +61,10 @@
     # Check that there aren't any cgroups anymore, neither any state records
     host.succeed(f"until [ ! -e {service}/nix-build@* ]; do sleep 1; done", timeout=30)
     host.succeed("until [ ! -e /nix/var/nix/cgroups/nix-build@* ]; do sleep 1; done", timeout=30)
+
+    # Check nested cgroup cleanup
+    logs = host.succeed("nix-build ${nestedCgroupsExpr nodes.host} 2>&1 | tee /dev/stderr")
+    assert "error: deleting cgroup" not in logs
   '';
 
 }