testlib: default nix fixture to also test all daemon protocols (2/3)

This commit enables the parametrization for "legacy-combined" protocol.
All failing tests with that have been either fixed or disabled. Notably:

- A couple of tests (e.g. involving builders) could be made to work with
a non-local store, but this would require some refactoring to testlib in
order to make certain configuration settings generic over local and
remote operations. We've disabled those for now, in order to make
progress
- The store tests all run locally only because of their nature
- The flakes tests also all run locally only because my energy for
fixing them is limited (and it's a *lot* of test failures, with probably
little overall benefit in terms of test coverage)

Change-Id: I56fa249a64f7c17c952f688ec89e9687f2a13f12

pyproject.toml

@@ -82,6 +82,7 @@ task-tags = ["TODO", "FIXME", "XXX"]
 #   TID251: ban certain APIs
 # TD: enforce TODO comment style
 #   disabled TD001: allow FIXME and XXX comments
+#   disabled TD002: author names on todos may be good, but simple fixmes dont need them
 #   disabled TD003: todos don't require an explicit issue link for us
 # ARG: disallow unused arguments
 # PTH: use pathlib instead of os calls
@@ -141,7 +142,7 @@ task-tags = ["TODO", "FIXME", "XXX"]
 # TRY: try and raise related things, not helpful as we only do testing
 # T20: Scripts usually output to stdout as either they are directly cli or passed into files
 select = ["E4", "E7", "E9", "F", "ERA", "ASYNC", "ANN0", "ANN2", "A", "C4", "ISC", "INP", "LOG", "G", "PIE", "PT", "Q", "RSE", "RET", "SIM", "TID251", "TD", "ARG", "PTH", "N", "PERF", "PLC", "PLE", "UP", "RUF"]
-ignore = ["ANN002", "ANN003", "TD001", "TD003", "PLE1", "RUF005", "RUF003"]
+ignore = ["ANN002", "ANN003", "TD001", "TD002", "TD003", "PLE1", "RUF005", "RUF003"]
 
 [tool.ruff.format]
 indent-style = "space"

tests/functional2/build/test_build_jobless.py

@@ -38,6 +38,7 @@ def test_j0_without_remotes_fails(nix: Nix):
     )
 
 
+@pytest.mark.no_daemon
 def test_j0_with_mismatched_remotes_fails(nix: Nix):
     remote = f"ssh://localhost?remote-store={nix.env.dirs.home}/machine1"
     result = nix.nix_build(["-j0", "--expr", drv, "--builders", remote]).run().expect(1)
@@ -47,6 +48,7 @@ def test_j0_with_mismatched_remotes_fails(nix: Nix):
     )
 
 
+@pytest.mark.no_daemon
 @with_files({"config.nix": get_global_asset("config.nix"), "default.nix": File(local_drv)})
 def test_j0_without_local_jobs(nix: Nix):
     result = nix.nix_build(["-j0", "--extra-local-jobs", "0"]).run().expect(1)

tests/functional2/build/test_ca.py

@@ -12,6 +12,9 @@ from typing import Literal
 
 from dataclasses import dataclass
 
+# FIXME: might be fixable by passing cert options to the daemon
+pytestmark = pytest.mark.no_daemon
+
 
 @dataclass
 class CertAssessment:

tests/functional2/build/test_fixed.py

@@ -28,7 +28,11 @@ def test_bad(nix: Nix):
 
     # Building with the bad hash should produce the "good" output path as
     # a sideeffect.
-    res = nix.nix_build(["fixed.nix", "-A", "bad", "--no-out-link"]).run().expect(102)
+    res = (
+        nix.nix_build(["fixed.nix", "-A", "bad", "--no-out-link"])
+        .run()
+        .expect(102 if not nix.uses_daemon else 1)
+    )
     assert "hash mismatch in fixed-output derivation" in res.stderr_plain
     assert path.exists()
 
@@ -80,7 +84,11 @@ def test_illegal_references(nix: Nix):
     - *-darwin has a different store location on the top of this in the sandbox (/private/tmp/...) causing further changes.
     Regex matching is the best we can afford.
     """
-    res = nix.nix_build(["fixed.nix", "-A", "illegalReferences"]).run().expect(102)
+    res = (
+        nix.nix_build(["fixed.nix", "-A", "illegalReferences"])
+        .run()
+        .expect(102 if not nix.uses_daemon else 1)
+    )
     assert re.findall(
         r"the fixed-output derivation '.*?/nix/store/[a-z0-9]*-illegal-reference.drv' must not reference store paths but 1 such references were found:.*/nix/store/[a-z0-9]*-fixed",
         res.stderr_plain,

tests/functional2/build/test_import_from_derivation.py

@@ -8,11 +8,16 @@ from testlib.utils import get_global_asset
 import re
 
 
-def filter_paths(nix: Nix, output: str) -> str:
+def filter_output(nix: Nix, output: str) -> str:
+    # Replace store hashes with something fixed, and also strip all daemon warnings about untrusted user settings
     return re.sub(
-        r"/nix/store/[0-9a-z]{32}",
-        "/nix/store/hashfilteredforrepeatableoutputs",
-        output.replace(nix.env.dirs.test_root.as_posix(), ""),
+        r"(?m)^warning:.*trusted user.*\n?",
+        "",
+        re.sub(
+            r"/nix/store/[0-9a-z]{32}",
+            "/nix/store/hashfilteredforrepeatableoutputs",
+            output.replace(nix.env.dirs.test_root.as_posix(), ""),
+        ),
     )
 
 
@@ -39,7 +44,7 @@ def test_warn_ifd(nix: Nix, snapshot: Callable[[str], Snapshot]):
         .stderr_plain
     )
 
-    assert snapshot("error") == filter_paths(nix, error)
+    assert snapshot("error") == filter_output(nix, error)
 
 
 @with_files(
@@ -64,7 +69,7 @@ def test_deny_ifd(nix: Nix, snapshot: Callable[[str], Snapshot]):
         .expect(1)
         .stderr_plain
     )
-    assert snapshot("error") == filter_paths(nix, error)
+    assert snapshot("error") == filter_output(nix, error)
 
 
 @with_files(
@@ -82,7 +87,7 @@ def test_allow_ifd(nix: Nix, snapshot: Callable[[str], Snapshot]):
     out_path = build.stdout_plain
 
     assert nix.physical_store_path_for(out_path).read_text() == "FOO579"
-    assert snapshot("error") == filter_paths(nix, error)
+    assert snapshot("error") == filter_output(nix, error)
 
 
 @with_files(
@@ -103,4 +108,4 @@ def test_instantiate_ifd_readonly_fail(nix: Nix, snapshot: Callable[[str], Snaps
         .expect(1)
         .stderr_plain
     )
-    assert snapshot("error") == filter_paths(nix, error)
+    assert snapshot("error") == filter_output(nix, error)

tests/functional2/build/test_remote.py

@@ -56,6 +56,7 @@ def _builders(proto: str, flags: list[str], env: ManagedEnv) -> str:
     """)
 
 
+@pytest.mark.no_daemon
 @pytest.mark.full_sandbox
 @with_files(
     {
@@ -98,6 +99,7 @@ def test_remote_trustless_unsigned(nix: Nix, env: ManagedEnv, busybox_args: list
     )
 
 
+@pytest.mark.no_daemon
 @pytest.mark.full_sandbox
 @pytest.mark.parametrize(
     ("protocol", "flags"), [("ssh", []), ("ssh-ng", []), ("ssh-ng", ["--force-untrusted"])]
@@ -127,6 +129,7 @@ def test_remote_trustless_ia(
     assert nix.physical_store_path_for(out_path).read_text() == "FOO BAR BAZ\n"
 
 
+@pytest.mark.no_daemon
 @pytest.mark.full_sandbox
 @pytest.mark.parametrize(("protocol", "flags"), [("ssh", []), ("ssh-ng", ["--force-untrusted"])])
 @with_files(

tests/functional2/build/test_substitution.py

@@ -51,18 +51,21 @@ def test_substitution_fallback_good_first(nix: Nix, caches: Caches):
     build(nix, "--substituters", f"{caches.good} {caches.bad}").ok()
 
 
+@pytest.mark.no_daemon
 def test_substitution_fallback_bad_first(nix: Nix, caches: Caches):
     # we expect three warnings for the single nar: two from querying, one from the substitution itself
     result = build(nix, "--substituters", f"{caches.bad} {caches.good}").ok()
     assert len(re.findall(r"warning.*narinfo", result.stderr_s)) == 3
 
 
+@pytest.mark.no_daemon
 def test_substitution_fallback_may_build(nix: Nix, caches: Caches):
     # we expect two errors for the single nar: one from querying, one from the substitution itself
     result = build(nix, "--substituters", f"{caches.bad}", "--fallback").ok()
     assert len(re.findall(r"error.*narinfo", result.stderr_s)) == 2
 
 
+@pytest.mark.no_daemon
 def test_substitution_fallback_no_build(nix: Nix, caches: Caches):
     # we expect one error, and it's fatal
     result = build(nix, "--substituters", f"{caches.bad}").expect(1)

tests/functional2/cli/test_daemon.py

@@ -2,6 +2,9 @@ from pathlib import Path
 from testlib.fixtures.nix import Nix
 from testlib.fixtures.file_helper import with_files
 from testlib.utils import get_global_asset_pack
+import pytest
+
+pytestmark = pytest.mark.no_daemon
 
 
 @with_files(get_global_asset_pack("simple-drv"))

tests/functional2/cli/test_store_ping.py

@@ -1,6 +1,8 @@
 import pytest
 from testlib.fixtures.nix import Nix, NixDaemon
 
+pytestmark = pytest.mark.no_daemon
+
 
 @pytest.fixture(autouse=True)
 def setup(nix: Nix):

tests/functional2/commands/test_build/test_build_fod.py

@@ -15,7 +15,11 @@ _build_args = ["build", "-f", "fod-failing.nix", "-L"]
 
 @with_files(_fod_files)
 def test_url_mismatch(nix: Nix):
-    res = nix.nix_build(["fod-failing.nix", "-A", "x1"]).run().expect(102)
+    res = (
+        nix.nix_build(["fod-failing.nix", "-A", "x1"])
+        .run()
+        .expect(102 if not nix.uses_daemon else 1)
+    )
     err = res.stderr_plain
     assert len(re.findall(r"hash mismatch in fixed-output derivation '.*-x1\.drv'", err)) == 1
     assert "likely URL: https://meow.puppy.forge/puppy.tar.gz" in err

tests/functional2/commands/test_build/test_timeout.py

@@ -1,6 +1,7 @@
 from testlib.fixtures.file_helper import with_files, CopyFile
 from testlib.fixtures.nix import Nix
 from testlib.utils import get_global_asset
+import pytest
 
 _files = {
     "timeout.nix": CopyFile("assets/test_timeout/timeout.nix"),
@@ -13,11 +14,12 @@ def test_timeout_timeout(nix: Nix):
     res = (
         nix.nix_build(["-Q", "timeout.nix", "-A", "infiniteLoop", "--timeout", "2"])
         .run()
-        .expect(101)
+        .expect(101 if not nix.uses_daemon else 1)
     )
     assert "timed out" in res.stderr_plain
 
 
+@pytest.mark.no_daemon
 @with_files(_files)
 def test_timeout_max_log(nix: Nix):
     res = (
@@ -30,13 +32,21 @@ def test_timeout_max_log(nix: Nix):
 
 @with_files(_files)
 def test_timeout_silent(nix: Nix):
-    res = nix.nix_build(["timeout.nix", "-A", "silent", "--max-silent-time", "2"]).run().expect(101)
+    res = (
+        nix.nix_build(["timeout.nix", "-A", "silent", "--max-silent-time", "2"])
+        .run()
+        .expect(101 if not nix.uses_daemon else 1)
+    )
     assert "file timed out after 2 seconds of silence" in res.stderr_plain
 
 
 @with_files(_files)
 def test_timeout_close_log(nix: Nix):
-    res = nix.nix_build(["timeout.nix", "-A", "closeLog"]).run().expect(100)
+    res = (
+        nix.nix_build(["timeout.nix", "-A", "closeLog"])
+        .run()
+        .expect(100 if not nix.uses_daemon else 1)
+    )
     assert "failed due to signal 9 (Killed" in res.stderr_plain
 
 

tests/functional2/daemon/test_connect.py

@@ -3,6 +3,8 @@ import itertools
 
 from testlib.fixtures.nix import Nix
 
+pytestmark = pytest.mark.no_daemon
+
 
 @pytest.fixture(autouse=True)
 def setup(nix: Nix):

tests/functional2/eval/test_eval_store.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from testlib.fixtures.file_helper import with_files
 from testlib.utils import get_global_asset_pack
 from testlib.fixtures.nix import Nix
+import pytest
 
 
 def assert_only_scratch_drvs(env: ManagedEnv):
@@ -12,6 +13,7 @@ def assert_only_scratch_drvs(env: ManagedEnv):
     assert list((env.dirs.home / "eval_store" / "nix" / "store").glob("*.drv"))
 
 
+@pytest.mark.no_daemon
 @with_files(get_global_asset_pack("dependencies"))
 def test_nix3_build(nix: Nix, files: Path):
     eval_store = files / "eval_store"
@@ -31,6 +33,7 @@ def test_nix_instantiate(nix: Nix, files: Path):
     assert_only_scratch_drvs(nix.env)
 
 
+@pytest.mark.no_daemon
 @with_files(get_global_asset_pack("dependencies"))
 def test_nix_build(nix: Nix, files: Path):
     res_link = files / "result"

tests/functional2/eval/test_fetch_mercurial.py

@@ -13,9 +13,10 @@ from testlib.fixtures.command import Command
 
 
 @pytest.fixture(autouse=True)
-def common_init(nix: Nix):
+def common_init(nix: Nix, env: ManagedEnv):
     nix.settings.add_xp_feature("nix-command")
     nix.env.path.add_program("hg")
+    env.path.add_program("hg")
 
 
 @dataclass

tests/functional2/eval/test_nix_path.py

@@ -5,6 +5,8 @@ from testlib.fixtures.file_helper import with_files
 from testlib.fixtures.nix import Nix
 from testlib.utils import get_global_asset
 
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files({"trivial.nix": get_global_asset("trivial.nix")})
 @pytest.mark.parametrize("prefix", ["by-absolute-path", "by-relative-path"])

tests/functional2/flakes/test_absolute_paths.py

@@ -3,6 +3,9 @@ from testlib.fixtures.git import Git
 from testlib.fixtures.file_helper import with_files, File
 from testlib.utils import get_global_asset_pack
 from pathlib import Path
+import pytest
+
+pytestmark = pytest.mark.no_daemon
 
 
 @with_files({"flake1": get_global_asset_pack(".git"), "input": File("input")})

tests/functional2/flakes/test_circular.py

@@ -4,6 +4,10 @@ from testlib.fixtures.file_helper import with_files, File
 from testlib.utils import get_global_asset_pack
 from pathlib import Path
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files(
     {

tests/functional2/flakes/test_cli.py

@@ -15,6 +15,8 @@ from testlib.fixtures.file_helper import with_files, File, FileDeclaration, _ini
 from testlib.environ import environ
 from .common import simple_flake, dependent_flake
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 
 

tests/functional2/flakes/test_config.py

@@ -6,6 +6,8 @@ from pathlib import Path
 from textwrap import dedent
 import pytest
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 
 files = get_global_asset_pack("simple-drv") | {

tests/functional2/flakes/test_develop.py

@@ -6,6 +6,8 @@ from testlib.utils import get_global_asset, get_global_asset_pack
 from pathlib import Path
 import pytest
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 
 flake = {

tests/functional2/flakes/test_flake_in_submodule.py

@@ -16,10 +16,12 @@ from testlib.fixtures.git import Git
 from testlib.fixtures.file_helper import with_files, File, EnvTemplate
 from testlib.utils import get_global_asset_pack
 from pathlib import Path
-import pytest
 import re
 import json
 import shutil
+import pytest
+
+pytestmark = pytest.mark.no_daemon
 
 root_without_self = get_global_asset_pack(".git") | {"root.nix": File('"expression in root repo"')}
 

tests/functional2/flakes/test_follow_paths.py

@@ -7,6 +7,8 @@ import pytest
 import fnmatch
 import json
 
+pytestmark = pytest.mark.no_daemon
+
 
 @pytest.fixture(autouse=True)
 def add_xp_features(nix: Nix):

tests/functional2/flakes/test_init.py

@@ -6,6 +6,8 @@ from testlib.utils import get_global_asset_pack, get_global_asset
 from pathlib import Path
 import pytest
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 
 files = {

tests/functional2/flakes/test_inputs.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from .common import simple_flake
 import pytest
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 files = simple_flake() | {
     "b-low": simple_flake()

tests/functional2/flakes/test_mercurial.py

@@ -9,6 +9,8 @@ from .common import simple_flake, dependent_flake
 from collections.abc import Callable
 import glob
 
+pytestmark = pytest.mark.no_daemon
+
 system = environ.get("system")
 
 files = {"flake-hg1": simple_flake(), "flake-hg2": dependent_flake()}

tests/functional2/flakes/test_subdir_flake.py

@@ -6,6 +6,10 @@ from testlib.utils import get_global_asset_pack
 from pathlib import Path
 from .common import simple_flake
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files({"flake-container": get_global_asset_pack(".git") | {"flake-dir": simple_flake()}})
 def test_subdir_flake(nix: Nix, files: Path, git: Git):

tests/functional2/flakes/test_symlink_paths.py

@@ -5,6 +5,10 @@ from testlib.fixtures.file_helper import CopyFile, Symlink, with_files, File
 from testlib.utils import get_global_asset_pack
 from testlib.fixtures.git import Git
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files(
     {"flake1": {"flake.nix": CopyFile("assets/flake1.nix")}, "flake1_sym": Symlink("flake1")}

tests/functional2/flakes/test_unlocked_override.py

@@ -4,6 +4,10 @@ from testlib.fixtures.file_helper import with_files, File
 from testlib.utils import get_global_asset_pack
 from pathlib import Path
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files(
     {

tests/functional2/store/cache/test_cache_compressions.py

@@ -5,6 +5,8 @@ from testlib.fixtures.nix import Nix
 from testlib.utils import get_global_asset_pack
 
 
+# FIXME: clear_store breaks daemons
+@pytest.mark.no_daemon
 @with_files(get_global_asset_pack("dependencies"))
 @pytest.mark.parametrize("algorithm", ["br", "zstd", "xz"])
 def test_cache_compressions(nix: Nix, algorithm: str):

tests/functional2/store/cache/test_substitute_truncated_nar.py

@@ -1,8 +1,11 @@
 from textwrap import dedent
 
 from testlib.fixtures.nix import Nix
+import pytest
 
 
+# FIXME: clear_store breaks daemons
+@pytest.mark.no_daemon
 def test_substitute_truncated_nar(nix: Nix):
     drv = dedent("""
         derivation {

tests/functional2/store/test_build.py

@@ -6,6 +6,10 @@ from testlib.fixtures.file_helper import with_files
 from testlib.fixtures.nix import Nix
 from testlib.utils import get_global_asset
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files({"config.nix": get_global_asset("config.nix")})
 def test_build_dir_permissions(nix: Nix):

tests/functional2/store/test_dump_db.py

@@ -4,6 +4,10 @@ from testlib.fixtures.file_helper import with_files
 from testlib.fixtures.nix import Nix
 from testlib.utils import get_global_asset_pack
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files(get_global_asset_pack("dependencies"))
 def test_dump_db(nix: Nix):

tests/functional2/store/test_evil_nars.py

@@ -16,6 +16,8 @@ from testlib.nar import (
     write_with_export_header,
 )
 
+pytestmark = pytest.mark.no_daemon
+
 meow_orig = "méow"
 meow_nfc_ = unicodedata.normalize("NFC", meow_orig)
 meow_nfd_ = unicodedata.normalize("NFD", meow_orig)

tests/functional2/store/test_export.py

@@ -3,6 +3,10 @@ from testlib.fixtures.command import Command
 from testlib.fixtures.file_helper import with_files
 from testlib.utils import get_global_asset_pack
 from testlib.fixtures.nix import Nix
+import pytest
+
+# FIXME: clear_store breaks daemons
+pytestmark = pytest.mark.no_daemon
 
 
 @with_files(get_global_asset_pack("dependencies"))

tests/functional2/store/test_http.py

@@ -8,6 +8,8 @@ from testlib.fixtures.file_helper import File, with_files
 from testlib.fixtures.http_server import http_server
 from testlib.fixtures.nix import Nix, with_diverted_store
 
+pytestmark = pytest.mark.no_daemon
+
 
 class HTTPStore:
     def __init__(self):

tests/functional2/store/test_optimise_store.py

@@ -4,6 +4,10 @@ from testlib.fixtures.file_helper import with_files
 from testlib.fixtures.nix import Nix, NixDaemon
 from testlib.utils import get_global_asset
 
+import pytest
+
+pytestmark = pytest.mark.no_daemon
+
 
 @with_files({"config.nix": get_global_asset("config.nix")})
 class TestOptimizeStore:

tests/functional2/testlib/fixtures/env.py

@@ -215,6 +215,8 @@ class ManagedEnv:
             # when writing things to the terminal (esp with man pages) use cat, to print the full output to stdout
             "PAGER": "cat",
             "BUILD_TEST_SHELL": self.shell_dir,
+            # asan can't print to stdio because that breaks snapshot tests
+            "ASAN_OPTIONS": f"log_path={self._tmp_path}/asan.log:{environ.get('ASAN_OPTIONS', '')}",
         }
         if self.build_env:
             self._env["BUILD_TEST_ENV"] = self.build_env

tests/functional2/testlib/fixtures/nix.py

@@ -445,7 +445,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
     if "nix" not in metafunc.fixturenames or "daemon" in metafunc.fixturenames:
         return
     if not list(metafunc.definition.iter_markers("no_daemon")):
-        protocols = [None]
+        protocols = [None, "legacy-combined"]
         # do not enable them the protocols for now
         # protocols += daemon_protocols # noqa ERA001
         ids = protocols

tests/functional2/testlib/fixtures/test_env.py

@@ -131,6 +131,7 @@ def test_env_to_env(tmp_path: Path):
         "BUILD_TEST_SHELL",
         "TMPDIR",
         "BUILD_TEST_ENV",
+        "ASAN_OPTIONS",
     } | ({"_NIX_TEST_NO_SANDBOX"} if sys.platform == "darwin" else set())