From 21dbd7745dd27d0ea92ca147d88fe1dca0de2dc3 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <raito@lix.systems>
Date: Mon, 9 Jun 2025 16:27:23 +0200
Subject: [PATCH] libstore/binary-cache-store: skip NAR listings if it's not
 possible to serialize it

Some source trees might not be representable inside of the NAR listing
format v1 as file paths (on Linux) are not guaranteed to be valid UTF-8.

When something like this happens on a large-scale build farm, a
mysterious "queued" but impossible to process job appears, this is
because we cannot write the NAR listing and serialization always fails.

Why did this work before? nlohmann was introduced _after_ such paths
were ingested, see: 09f00dd4d01aa1b6866978d162022133e521614f.

What happened for such previously mis-serialized NAR listings?

```
curl -v 'https://cache.nixos.org/nz8p9hn00r6z7s57581c1hiv39pa1ia6.ls' |
brotli -d | jq .
```

This fixes the build of `sub-batch`
(https://github.com/kl/sub-batch/tree/master/tests/rename_invalid_utf8)
on ForkOS infrastructure.

Many thanks to Puck for the assistance on holding `rr` right on this one
and finding the history of these changes.

Change-Id: I2c2fbac70818e02810f9fd236c3a248187bf5fe7
Signed-off-by: Raito Bezarius <raito@lix.systems>
---
 lix/libstore/binary-cache-store.cc | 14 +++++++++++++-
 tests/functional/nar-access.nix    |  3 +++
 tests/functional/nar-access.sh     |  7 +++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc
index 1a67ec27e..3cab26714 100644
--- a/lix/libstore/binary-cache-store.cc
+++ b/lix/libstore/binary-cache-store.cc
@@ -178,7 +178,19 @@ try {
             {"root", listNar(narIndex)},
         };
 
-        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
+        try {
+            upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
+        } catch (ForeignException & exc) {
+            if (exc.is<JSON::exception>()) {
+                warn(
+                    "Skipping NAR listing for path '%1%' due to serialization failure: %2%",
+                    printStorePath(narInfo->path),
+                    exc.what()
+                );
+            } else {
+                throw exc;
+            }
+        }
     }
 
     /* Optionally maintain an index of DWARF debug info files
diff --git a/tests/functional/nar-access.nix b/tests/functional/nar-access.nix
index 9948abe59..6756291b4 100644
--- a/tests/functional/nar-access.nix
+++ b/tests/functional/nar-access.nix
@@ -1,3 +1,4 @@
+{ nonUtf8Inodes ? false }:
 with import ./config.nix;
 
 rec {
@@ -13,6 +14,8 @@ rec {
         touch $out/qux
         mkdir $out/zyx
 
+        ${if nonUtf8Inodes then ''printf "data" > "$out/invalid-\x80file"'' else ""}
+
         cat >$out/foo/data <<EOF
         lasjdöaxnasd
 asdom 12398
diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh
index af22a87eb..7667a2717 100644
--- a/tests/functional/nar-access.sh
+++ b/tests/functional/nar-access.sh
@@ -2,6 +2,7 @@ source common.sh
 
 echo "building test path"
 storePath="$(nix-build nar-access.nix -A a --no-out-link)"
+strangerStorePath="$(nix-build nar-access.nix -A a --arg nonUtf8Inode true --no-out-link)"
 
 cd "$TEST_ROOT"
 
@@ -79,3 +80,9 @@ rm -rf $cacheDir/nar
 diff -u \
     <(nix store ls --json -R $storePath/foo/bar --store "file://$cacheDir" | jq -S) \
     <(echo '{"narOffset": 368,"type":"regular","size":0}' | jq -S)
+
+# Confirm that there's no more than one `.ls` in the `$cacheDir` because non-UTF8 inodes cannot have `.ls` generated for them.
+[[ $(find $cacheDir -type f -name '*.ls' | wc -l) -eq 1 ]] || (echo "Expected at most one listing file in $cacheDir, found more"; exit -1)
+
+# Confirm that NARs with non-UTF8 inodes can still be listed
+expect 0 nix store ls $strangerStorePath/ --store "file://$cacheDir"