Raito Bezarius
21dbd7745d
Some source trees might not be representable inside of the NAR listing
format v1 as file paths (on Linux) are not guaranteed to be valid UTF-8.
When something like this happens on a large-scale build farm, a
mysterious "queued" but impossible to process job appears, this is
because we cannot write the NAR listing and serialization always fails.
Why did this work before? nlohmann was introduced _after_ such paths
were ingested, see: 09f00dd4d0.
What happened for such previously mis-serialized NAR listings?
```
curl -v 'https://cache.nixos.org/nz8p9hn00r6z7s57581c1hiv39pa1ia6.ls' |
brotli -d | jq .
```
This fixes the build of `sub-batch`
(https://github.com/kl/sub-batch/tree/master/tests/rename_invalid_utf8)
on ForkOS infrastructure.
Many thanks to Puck for the assistance on holding `rr` right on this one
and finding the history of these changes.
Change-Id: I2c2fbac70818e02810f9fd236c3a248187bf5fe7
Signed-off-by: Raito Bezarius <raito@lix.systems>
89 lines
3.4 KiB
Bash
89 lines
3.4 KiB
Bash
source common.sh
|
|
|
|
echo "building test path"
|
|
storePath="$(nix-build nar-access.nix -A a --no-out-link)"
|
|
strangerStorePath="$(nix-build nar-access.nix -A a --arg nonUtf8Inode true --no-out-link)"
|
|
|
|
cd "$TEST_ROOT"
|
|
|
|
# Dump path to nar.
|
|
narFile="$TEST_ROOT/path.nar"
|
|
nix-store --dump $storePath > $narFile
|
|
|
|
# Check that find and nar ls match.
|
|
( cd $storePath; find . | sort ) > files.find
|
|
nix nar ls -R -d $narFile "" | sort > files.ls-nar
|
|
diff -u files.find files.ls-nar
|
|
|
|
# Check that file contents of data match.
|
|
nix nar cat $narFile /foo/data > data.cat-nar
|
|
diff -u data.cat-nar $storePath/foo/data
|
|
|
|
# Check that file contents of baz match.
|
|
nix nar cat $narFile /foo/baz > baz.cat-nar
|
|
diff -u baz.cat-nar $storePath/foo/baz
|
|
|
|
nix store cat $storePath/foo/baz > baz.cat-nar
|
|
diff -u baz.cat-nar $storePath/foo/baz
|
|
|
|
# Check that 'nix store cat' fails on invalid store paths.
|
|
invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
|
|
cp -r $storePath $invalidPath
|
|
expect 1 nix store cat $invalidPath/foo/baz
|
|
|
|
# Test --json.
|
|
diff -u \
|
|
<(nix nar ls --json $narFile / | jq -S) \
|
|
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
|
|
diff -u \
|
|
<(nix nar ls --json -R $narFile /foo | jq -S) \
|
|
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
|
|
diff -u \
|
|
<(nix nar ls --json -R $narFile /foo/bar | jq -S) \
|
|
<(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
|
|
diff -u \
|
|
<(nix store ls --json $storePath | jq -S) \
|
|
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
|
|
diff -u \
|
|
<(nix store ls --json -R $storePath/foo | jq -S) \
|
|
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
|
|
diff -u \
|
|
<(nix store ls --json -R $storePath/foo/bar| jq -S) \
|
|
<(echo '{"type":"regular","size":0}' | jq -S)
|
|
|
|
# Test missing files.
|
|
expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
|
|
expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist'
|
|
|
|
# Test failure to dump.
|
|
if nix-store --dump $storePath >/dev/full ; then
|
|
echo "dumping to /dev/full should fail"
|
|
exit -1
|
|
fi
|
|
|
|
|
|
# Test reading from remote nar listings if available
|
|
|
|
nix copy --to "file://$cacheDir?write-nar-listing=true" $storePath
|
|
|
|
export _NIX_FORCE_HTTP=1
|
|
|
|
diff -u \
|
|
<(nix store ls --json $storePath --store "file://$cacheDir" | jq -S) \
|
|
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
|
|
diff -u \
|
|
<(nix store ls --json -R $storePath/foo/bar --store "file://$cacheDir" | jq -S) \
|
|
<(echo '{"narOffset": 368,"type":"regular","size":0}' | jq -S)
|
|
|
|
# Confirm that we are reading from ".ls" file by deleting the nar
|
|
rm -rf $cacheDir/nar
|
|
diff -u \
|
|
<(nix store ls --json -R $storePath/foo/bar --store "file://$cacheDir" | jq -S) \
|
|
<(echo '{"narOffset": 368,"type":"regular","size":0}' | jq -S)
|
|
|
|
# Confirm that there's no more than one `.ls` in the `$cacheDir` because non-UTF8 inodes cannot have `.ls` generated for them.
|
|
[[ $(find $cacheDir -type f -name '*.ls' | wc -l) -eq 1 ]] || (echo "Expected at most one listing file in $cacheDir, found more"; exit -1)
|
|
|
|
# Confirm that NARs with non-UTF8 inodes can still be listed
|
|
expect 0 nix store ls $strangerStorePath/ --store "file://$cacheDir"
|