From ac8209404cb1b6b89ad3ee527985ac2e7ad7e253 Mon Sep 17 00:00:00 2001
From: Alois Wohlschlager <alois1@gmx-topmail.de>
Date: Tue, 14 Oct 2025 11:48:22 +0200
Subject: [PATCH] nix-eval-jobs: estimate memory usage using the heap size

When the maximum memory size is chosen too large (relative to the free memory
usage), so that the evaluation workers don't fit into the free memory, they
will start being swapped out. Further increase in memory usage will not be
reflected in the RSS, so they will not exit, and instead fill up all swap too
until the system runs out of memory.

The BDW-GC keeps track of its heap size, including any parts of it that may be
swapped out. For this reason it does not suffer from a similar problem. While
not all memory usage by the evaluation worker is accounted for by the GC, a
simple affine model for the overhead works reasonably well in practice. The
specific parameter values were determined using nixpkgs evaluation, and appear
to work quite well also for somewhat different workloads (like the `hydraJobs`
of Lix, which include a bunch of NixOS system configurations). So this is what
gets used.

When Lix is configured without the BDW-GC, of course this approach cannot be
used. In this case the old strategy is retained.

Change-Id: I6cb5f0a9b4ceda9dd14be165dda108cd6a6a6964
---
 .../nix-eval-jobs/src/nix-eval-jobs.cc        |  4 ---
 subprojects/nix-eval-jobs/src/worker.cc       | 27 ++++++++++++++++---
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc
index d7a2f1448..100e6ec7f 100644
--- a/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc
+++ b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc
@@ -334,10 +334,6 @@ void collector(MyArgs &myArgs, Sync<State> &state_,
 }
 
 int main(int argc, char **argv) {
-
-    /* We are doing the garbage collection by killing forks */
-    setenv("GC_DONT_GC", "1", 1);
-
     return handleExceptions(argv[0], [&]() {
         initNix();
         initLibExpr();
diff --git a/subprojects/nix-eval-jobs/src/worker.cc b/subprojects/nix-eval-jobs/src/worker.cc
index b644a0bbe..138d81b3f 100644
--- a/subprojects/nix-eval-jobs/src/worker.cc
+++ b/subprojects/nix-eval-jobs/src/worker.cc
@@ -124,6 +124,20 @@ readConstituents(const nix::Value *v, nix::box_ptr<nix::EvalState> &state,
 
 void worker(nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args)
 try {
+#if HAVE_BOEHMGC
+    // We are doing the garbage collection by killing forks.
+    GC_disable();
+
+    // There is some memory usage overhead on top of the GC heap size.
+    // A simple model using fixed and proportional overhead already gives reasonable results.
+    // The parameters have been determined experimentally using evaluation of nixpkgs.
+    size_t maxGcHeapSize = std::ldexp(std::max(0.0, 0.9 * (static_cast<double>(args.maxMemorySize) - 150.0)), 20);
+    if (GC_get_heap_size() > maxGcHeapSize) {
+        throw nix::Error("The heap is too large. Increase %s or reduce %s.",
+                         "--max-memory-size", "GC_INITIAL_HEAP_SIZE");
+    }
+#endif
+
     nix::AsyncIoRoot aio;
 
     auto evalStore = aio.blockOn(args.evalStoreUrl
@@ -250,12 +264,19 @@ try {
             return; // main process died
         }
 
-        /* If our RSS exceeds the maximum, exit. The collector will
-           start a new process. */
+        // If the memory limit is exceeded, exit.
+        // The collector will start a new process.
+#if HAVE_BOEHMGC
+        if (GC_get_heap_size() > maxGcHeapSize) {
+            break;
+        }
+#else
         struct rusage r;
         getrusage(RUSAGE_SELF, &r);
-        if ((size_t)r.ru_maxrss > args.maxMemorySize * 1024)
+        if ((size_t)r.ru_maxrss > args.maxMemorySize * 1024) {
             break;
+        }
+#endif
     }
 
     if (tryWriteLine(to.get(), "restart") < 0) {