From 95f15cf94fb621795eb854db6d0387cfd22cd310 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Thu, 30 Oct 2025 00:01:03 +0100 Subject: [PATCH] libstore/binary-cache: default to zstd for compression libarchive's xz offers single threaded xz compression which is very slow and provides ~10-20Mbps compression speed in addition to maxing a core. In exchange, it achieves optimal compression ratios among all our compression methods. Nonetheless, xz prevent the saturation of 1Gbps+ connections and slow down significantly decompression for end users. As these connections and faster hardware is becoming prevalent for cache servers and clients, we offer to default to zstd. Lix is a "compress once, decompress many times" application. To avoid incurring a high penalty to end users very sensitive to compress ratio (very slow Internet connections), we dampen the consequences of switching to zstd by increasing the default zstd level to 12. On one example, xz will compress a 4.4GB file to 632MB, zstd on 12 will compress it to 775MB, that is a ~18 % increase over the optimal xz compression. zstd took 18 seconds to produce this file. Increasing to level 14 leads to a 773MB file while taking 37s. Increasing to level 16 leads to 735MB file while taking 66s. Finally, xz took 77s, so a 50 % reduction in time taken to compress in exchange of an increase of 18 % of the compressed size. This change will reduce issues encountered in #945 but is probably not the root cause. References: - https://discourse.nixos.org/t/switch-cache-nixos-org-to-zstd-to-fix-slow-nixos-updates-nix-downloads/23961 Change-Id: I7beda2bf2c1fed146dcb797b8f85dc290c486ab2 Signed-off-by: Raito Bezarius --- .../rl-next/enable-zstd-for-binary-cache.md | 36 +++++++++++++++++++ lix/libstore/binary-cache-store.hh | 19 +++++++--- lix/libutil/compression.cc | 11 ++++++ package.nix | 2 ++ tests/functional/binary-cache.sh | 4 +-- tests/functional/substitute-truncated-nar.sh | 6 ++-- 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 doc/manual/rl-next/enable-zstd-for-binary-cache.md diff --git a/doc/manual/rl-next/enable-zstd-for-binary-cache.md b/doc/manual/rl-next/enable-zstd-for-binary-cache.md new file mode 100644 index 000000000..125ee4951 --- /dev/null +++ b/doc/manual/rl-next/enable-zstd-for-binary-cache.md @@ -0,0 +1,36 @@ +--- +synopsis: Enable high compress ratio zstd compression by default for binary caches uploads +issues: [fj#945] +cls: [4503] +category: "Breaking Changes" +credits: [horrors, raito] +--- + +The default compression method for binary cache uploads has been switched from +[`xz`](https://github.com/tukaani-project/xz) to +[`zstd`](https://github.com/facebook/zstd) to address performance and usability +issues related to modern hardware and high-speed connections. + +## Why? + +`xz` offers compression ratios but is single-threaded in our implementation and +very slow (~10-20 Mbps in our test), preventing full utilization of 100Mbps+ +connections and significantly slowing decompression for end users. + +Lix is a "compress once, decompress many" application: build farms can afford +to spend more time compressing to achieve a faster download transfer for the +end user. More importantly, it matters that all end users spend the least +amount of time decompressing. + +## What about compression ratios? + +`zstd` cannot achieve the same peaks as `xz`, nonetheless, `zstd` compression +level has been increased to level 12 by default to balance compression ratio +and performance. + +## Synthetic test case data + +* **xz** (default compression level) on a 4.4GB file: ~632MB (77s) +* **zstd** (level 12) on the same file: ~775MB (18s), 18% larger but 50% faster +* **zstd** (level 14): ~773MB (37s) +* **zstd** (level 16): ~735MB (66s) diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index e8a7e4c5d..40cb3056c 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -19,8 +19,12 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + const Setting compression{ + this, + "zstd", + "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)." + }; const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; @@ -40,12 +44,19 @@ struct BinaryCacheStoreConfig : virtual StoreConfig const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{this, -1, "compression-level", + const Setting compressionLevel{ + this, + -1, + "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. `-1` specifies that the default compression level should be used. - )"}; + + Note: when using zstd `-1` will select level 12 to approximately match xz compression + ratios at default settings rather than the zstd library default of 3. + )" + }; }; diff --git a/lix/libutil/compression.cc b/lix/libutil/compression.cc index eea659c10..d4ed63a0f 100644 --- a/lix/libutil/compression.cc +++ b/lix/libutil/compression.cc @@ -465,6 +465,17 @@ ref makeCompressionSink(const std::string & method, Sink & next std::vector la_supports = { "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" }; + + // NOTE: Lix overrides the default here because we want the default zstd behavior + // to perform well on compression ratios in the hopes to approach what xz provided in the past. + // We choose one that is much faster than xz while being in range of xz compression ratios. + // + // In our experience, further levels provides marginal benefits but makes the compression speed + // much slower in exchange. + if (level == COMPRESSION_LEVEL_DEFAULT && method == "zstd") { + level = 12; + } + if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/package.nix b/package.nix index cdc7f1228..fb86ce844 100644 --- a/package.nix +++ b/package.nix @@ -61,6 +61,7 @@ util-linuxMinimal ? utillinuxMinimal, utillinuxMinimal ? null, xz, + zstd, yq, busybox-sandbox-shell, @@ -331,6 +332,7 @@ stdenv.mkDerivation (finalAttrs: { jq yq lsof + zstd ] ++ lib.optional hostPlatform.isLinux util-linuxMinimal ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 50e23e56c..afd61e692 100644 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -73,10 +73,10 @@ basicDownloadTests # Test whether Lix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) +nar=$(ls $cacheDir/nar/*.nar.zst | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nix-store --dump $TEST_ROOT/empty | zstd - --stdout > $nar expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log grepQuiet "hash mismatch" $TEST_ROOT/log diff --git a/tests/functional/substitute-truncated-nar.sh b/tests/functional/substitute-truncated-nar.sh index 1ac7efaf6..cb8188638 100644 --- a/tests/functional/substitute-truncated-nar.sh +++ b/tests/functional/substitute-truncated-nar.sh @@ -15,13 +15,13 @@ path=$(build) nix copy --to "$BINARY_CACHE" "$path" nix-collect-garbage >/dev/null 2>&1 -nar=0bylmx35yjy2b1b4k7gjsl7i4vc03cpmryb41grfb1mp40n3hifl.nar.xz +nar=0c3y7p42issm0ydjilwvk0drv958p4p4d2d6c7y5ksmzmbf7rfhg.nar.zst [ -e $cacheDir/nar/$nar ] || fail "long nar missing?" -xzcat $cacheDir/nar/$nar > $TEST_HOME/tmp +zstdcat $cacheDir/nar/$nar > $TEST_HOME/tmp truncate -s $(( $(stat -c %s $TEST_HOME/tmp) - 10 )) $TEST_HOME/tmp -xz < $TEST_HOME/tmp > $cacheDir/nar/$nar +zstd - --stdout < $TEST_HOME/tmp > $cacheDir/nar/$nar # Copying back '$path' from the binary cache. This should fail as it is truncated if build --option substituters "$BINARY_CACHE" --option require-sigs false -j0; then