diff --git a/doc/manual/rl-next/enable-zstd-for-binary-cache.md b/doc/manual/rl-next/enable-zstd-for-binary-cache.md new file mode 100644 index 000000000..125ee4951 --- /dev/null +++ b/doc/manual/rl-next/enable-zstd-for-binary-cache.md @@ -0,0 +1,36 @@ +--- +synopsis: Enable high compress ratio zstd compression by default for binary caches uploads +issues: [fj#945] +cls: [4503] +category: "Breaking Changes" +credits: [horrors, raito] +--- + +The default compression method for binary cache uploads has been switched from +[`xz`](https://github.com/tukaani-project/xz) to +[`zstd`](https://github.com/facebook/zstd) to address performance and usability +issues related to modern hardware and high-speed connections. + +## Why? + +`xz` offers compression ratios but is single-threaded in our implementation and +very slow (~10-20 Mbps in our test), preventing full utilization of 100Mbps+ +connections and significantly slowing decompression for end users. + +Lix is a "compress once, decompress many" application: build farms can afford +to spend more time compressing to achieve a faster download transfer for the +end user. More importantly, it matters that all end users spend the least +amount of time decompressing. + +## What about compression ratios? + +`zstd` cannot achieve the same peaks as `xz`, nonetheless, `zstd` compression +level has been increased to level 12 by default to balance compression ratio +and performance. + +## Synthetic test case data + +* **xz** (default compression level) on a 4.4GB file: ~632MB (77s) +* **zstd** (level 12) on the same file: ~775MB (18s), 18% larger but 50% faster +* **zstd** (level 14): ~773MB (37s) +* **zstd** (level 16): ~735MB (66s) diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index e8a7e4c5d..40cb3056c 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -19,8 +19,12 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + const Setting compression{ + this, + "zstd", + "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)." + }; const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; @@ -40,12 +44,19 @@ struct BinaryCacheStoreConfig : virtual StoreConfig const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{this, -1, "compression-level", + const Setting compressionLevel{ + this, + -1, + "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. `-1` specifies that the default compression level should be used. - )"}; + + Note: when using zstd `-1` will select level 12 to approximately match xz compression + ratios at default settings rather than the zstd library default of 3. + )" + }; }; diff --git a/lix/libutil/compression.cc b/lix/libutil/compression.cc index eea659c10..d4ed63a0f 100644 --- a/lix/libutil/compression.cc +++ b/lix/libutil/compression.cc @@ -465,6 +465,17 @@ ref makeCompressionSink(const std::string & method, Sink & next std::vector la_supports = { "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" }; + + // NOTE: Lix overrides the default here because we want the default zstd behavior + // to perform well on compression ratios in the hopes to approach what xz provided in the past. + // We choose one that is much faster than xz while being in range of xz compression ratios. + // + // In our experience, further levels provides marginal benefits but makes the compression speed + // much slower in exchange. + if (level == COMPRESSION_LEVEL_DEFAULT && method == "zstd") { + level = 12; + } + if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/package.nix b/package.nix index cdc7f1228..fb86ce844 100644 --- a/package.nix +++ b/package.nix @@ -61,6 +61,7 @@ util-linuxMinimal ? utillinuxMinimal, utillinuxMinimal ? null, xz, + zstd, yq, busybox-sandbox-shell, @@ -331,6 +332,7 @@ stdenv.mkDerivation (finalAttrs: { jq yq lsof + zstd ] ++ lib.optional hostPlatform.isLinux util-linuxMinimal ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 50e23e56c..afd61e692 100644 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -73,10 +73,10 @@ basicDownloadTests # Test whether Lix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) +nar=$(ls $cacheDir/nar/*.nar.zst | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nix-store --dump $TEST_ROOT/empty | zstd - --stdout > $nar expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log grepQuiet "hash mismatch" $TEST_ROOT/log diff --git a/tests/functional/substitute-truncated-nar.sh b/tests/functional/substitute-truncated-nar.sh index 1ac7efaf6..cb8188638 100644 --- a/tests/functional/substitute-truncated-nar.sh +++ b/tests/functional/substitute-truncated-nar.sh @@ -15,13 +15,13 @@ path=$(build) nix copy --to "$BINARY_CACHE" "$path" nix-collect-garbage >/dev/null 2>&1 -nar=0bylmx35yjy2b1b4k7gjsl7i4vc03cpmryb41grfb1mp40n3hifl.nar.xz +nar=0c3y7p42issm0ydjilwvk0drv958p4p4d2d6c7y5ksmzmbf7rfhg.nar.zst [ -e $cacheDir/nar/$nar ] || fail "long nar missing?" -xzcat $cacheDir/nar/$nar > $TEST_HOME/tmp +zstdcat $cacheDir/nar/$nar > $TEST_HOME/tmp truncate -s $(( $(stat -c %s $TEST_HOME/tmp) - 10 )) $TEST_HOME/tmp -xz < $TEST_HOME/tmp > $cacheDir/nar/$nar +zstd - --stdout < $TEST_HOME/tmp > $cacheDir/nar/$nar # Copying back '$path' from the binary cache. This should fail as it is truncated if build --option substituters "$BINARY_CACHE" --option require-sigs false -j0; then