diff --git a/tests/functional2/testlib/fixtures/snapshot.py b/tests/functional2/testlib/fixtures/snapshot.py index f2ddd14d6..38f4dfdad 100644 --- a/tests/functional2/testlib/fixtures/snapshot.py +++ b/tests/functional2/testlib/fixtures/snapshot.py @@ -3,6 +3,7 @@ from collections.abc import Callable from logging import Logger from pathlib import Path from typing import Any +from collections.abc import Generator import pytest from _pytest.fixtures import FixtureRequest @@ -22,6 +23,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) +@pytest.fixture(scope="session") +def do_snapshot_update(request: FixtureRequest) -> bool: + return request.config.getoption("accept-tests") or ( + os.environ.get("_NIX_TEST_ACCEPT") is not None + ) + + class Snapshot: def __init__(self, expected_output_path: Path, do_update: bool, logger: Logger): """ @@ -40,6 +48,7 @@ class Snapshot: self.content = expected_output_path.read_text() if expected_output_path.exists() else "" self.do_update = do_update self.logger = logger + self.did_update = False def __eq__(self, other: Any) -> bool: """ @@ -56,6 +65,7 @@ class Snapshot: "snapshot didn't propagate, the updated file can be found here: %s", self.expected_output_path.absolute(), ) + self.did_update = True return True return are_equal @@ -67,7 +77,9 @@ class Snapshot: @pytest.fixture -def snapshot(request: FixtureRequest, logger: Logger, tmp_path: Path) -> Callable[[str], Snapshot]: +def snapshot( + do_snapshot_update: bool, logger: Logger, tmp_path: Path +) -> Generator[Callable[[str], Snapshot], Any, None]: """ create a snapshot for the given output file the snapshot must be the **left hand operator** of the equal check. @@ -82,17 +94,18 @@ def snapshot(request: FixtureRequest, logger: Logger, tmp_path: Path) -> Callabl :return: a snapshot object which one can use `==` on """ + snaps: list[Snapshot] = [] + def create_snapshot(expected_output_path: str) -> Snapshot: exp_path = tmp_path / expected_output_path if not exp_path.is_symlink(): logger.warning( "expected output file isn't a symlink. When accepting the tests output, the update might not propagate" ) - return Snapshot( - exp_path, - request.config.getoption("accept-tests") - or os.environ.get("_NIX_TEST_ACCEPT") is not None, - logger, - ) + obj = Snapshot(exp_path, do_snapshot_update, logger) + snaps.append(obj) + return obj - return create_snapshot + yield create_snapshot + if any(s.did_update for s in snaps): + pytest.skip("Updated Golden Files") diff --git a/tests/functional2/testlib/fixtures/test_snapshot.py b/tests/functional2/testlib/fixtures/test_snapshot.py new file mode 100644 index 000000000..4e85ff868 --- /dev/null +++ b/tests/functional2/testlib/fixtures/test_snapshot.py @@ -0,0 +1,271 @@ +import logging +from collections.abc import Callable +from pathlib import Path +from textwrap import dedent + +import pytest +from _pytest.logging import LogCaptureFixture +from functional2.testlib.commands import Command +from functional2.testlib.fixtures.file_helper import ( + CopyFile, + File, + FileDeclaration, + merge_file_declaration, +) +from functional2.testlib.fixtures.snapshot import Snapshot +from functional2.testlib.utils import get_functional2_files + + +def _get_f2_snapshot_files(additional_files: FileDeclaration) -> FileDeclaration: + return get_functional2_files( + merge_file_declaration( + { + "functional2": { + "testlib": { + "__init__.py": CopyFile("../__init__.py"), + "fixtures": { + "__init__.py": CopyFile("__init__.py"), + "snapshot.py": CopyFile("snapshot.py"), + "logger.py": CopyFile("logger.py"), + }, + }, + "conftest.py": File( + "pytest_plugins = ('functional2.testlib.fixtures.logger', 'functional2.testlib.fixtures.snapshot')" + ), + } + }, + additional_files, + ) + ) + + +def test_snapshot_empty_on_no_file(snapshot: Callable[[str], Snapshot]): + # no qa here, as we want to exactly check for empty string + assert snapshot("this_file_does_not_exist") == "" # noqa: PLC1901 + + +@pytest.mark.parametrize("files", [{"empty_file.txt": File("")}], indirect=True) +@pytest.mark.usefixtures("files") +def test_snapshot_empty_on_empty_file(snapshot: Callable[[str], Snapshot]): + # no qa here, as we want to exactly check for empty string + assert snapshot("empty_file.txt") == "" # noqa: PLC1901 + + +@pytest.mark.parametrize("files", [{"out.exp": File("Hell o' World")}], indirect=True) +@pytest.mark.usefixtures("files") +def test_snapshot_warns_on_non_symlink_file( + snapshot: Callable[[str], Snapshot], caplog: LogCaptureFixture +): + with caplog.at_level(logging.WARNING): + snapshot("out.exp") + assert len(caplog.records) == 1 + assert ( + caplog.records[0].msg + == "expected output file isn't a symlink. When accepting the tests output, the update might not propagate" + ) + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + _get_f2_snapshot_files( + { + "functional2": { + "test_snapshot": { + "test_snapshot.py": File( + dedent(""" + def test_noupdate(do_snapshot_update): + assert not do_snapshot_update + """) + ) + } + } + } + ), + ([], False), + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_do_update_false_when_none_set(pytest_command: Command): + pytest_command.run().ok() + + +_update_test_files = _get_f2_snapshot_files( + { + "functional2": { + "test_snapshot": { + "test_snapshot.py": File( + dedent(""" + def test_does_update(do_snapshot_update): + assert do_snapshot_update + """) + ) + } + } + } +) + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [(_update_test_files, ([], False)), (_update_test_files, (["--accept-tests"], False))], + indirect=True, +) +@pytest.mark.parametrize("set_env", [False, True]) +@pytest.mark.usefixtures("files") +def test_do_update_true_when_any_set(pytest_command: Command, set_env: bool): + if not (set_env or "--accept-tests" in pytest_command.argv): + pytest.skip("not this test case") + if set_env: + pytest_command.update_env(_NIX_TEST_ACCEPT="1") + pytest_command.run().ok() + + +def _snapshot_test_files(content: str) -> FileDeclaration: + return _get_f2_snapshot_files( + { + "functional2": { + "test_snapshot": { + "test_snapshot.py": File( + dedent(f""" + def test_snapshot(snapshot, tmp_path): + (tmp_path / "out.exp").write_text("{content}") + assert snapshot("out.exp") == "plush plush" + """) + ) + } + } + } + ) + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [(_snapshot_test_files("plush plush"), (["--accept-tests"], False))], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_snapshot_no_updated_when_equal(pytest_command: Command): + res = pytest_command.run().ok() + assert "the updated file can be found here" not in res.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), [(_snapshot_test_files("fops plush"), ([], False))], indirect=True +) +@pytest.mark.usefixtures("files") +def test_snapshot_fails_on_diff(pytest_command: Command): + res = pytest_command.run().expect(1) + assert "FAILED test_snapshot/test_snapshot.py::test_snapshot" in res.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [(_snapshot_test_files("fops plush"), (["--accept-tests"], False))], + indirect=True, +) +def test_snapshot_updates_diff(files: Path, pytest_command: Command): + output_file = files / "pytest_files/test_snapshot0/out.exp" + pytest_command.run().ok() + assert output_file.read_text() == "plush plush" + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [(_snapshot_test_files("fops plush"), (["--accept-tests"], False))], + indirect=True, +) +def test_snapshot_updates_shares_updated_location_without_symlink( + files: Path, pytest_command: Command +): + expected_path = "pytest_files/test_snapshot0/out.exp" + output_file = files / expected_path + res = pytest_command.run().ok() + assert output_file.read_text() == "plush plush" + assert "the updated file can be found here" in res.stdout_plain + assert expected_path in res.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [(_snapshot_test_files("fops plush"), (["--accept-tests"], False))], + indirect=True, +) +def test_snapshot_marks_skip_after_update(files: Path, pytest_command: Command): + expected_path = "pytest_files/test_snapshot0/out.exp" + output_file = files / expected_path + res = pytest_command.run().ok() + assert output_file.read_text() == "plush plush" + assert "test_snapshot/test_snapshot.py::test_snapshot SKIPPED (Updated" in res.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + _get_f2_snapshot_files( + { + "functional2": { + "test_snapshot": { + "test_snapshot.py": File( + dedent(""" + def test_snapshot(snapshot, tmp_path): + (tmp_path / "out.exp").write_text("fops plush") + (tmp_path / "err.exp").write_text("shork plush") + assert snapshot("out.exp") == "plush plush" + assert snapshot("err.exp") == "blobhaj" + """) + ) + } + } + } + ), + (["--accept-tests"], False), + ) + ], + indirect=True, +) +def test_snapshot_updates_multiple(files: Path, pytest_command: Command): + expected_path = "pytest_files/test_snapshot0" + first_file = files / expected_path / "out.exp" + second_file = files / expected_path / "err.exp" + res = pytest_command.run().ok() + assert "test_snapshot/test_snapshot.py::test_snapshot SKIPPED (Updated" in res.stdout_plain + assert first_file.read_text() == "plush plush" + assert second_file.read_text() == "blobhaj" + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + _get_f2_snapshot_files( + { + "functional2": { + "test_snapshot": { + "test_snapshot.py": File( + dedent(""" + def test_snapshot(snapshot, tmp_path): + (tmp_path / "updated.txt").write_text("snek plush") + (tmp_path / "out.exp").symlink_to("./updated.txt") + assert snapshot("out.exp") == "plush plush" + """) + ) + } + } + } + ), + (["--accept-tests"], False), + ) + ], + indirect=True, +) +def test_snapshot_updates_no_location_when_symlink(files: Path, pytest_command: Command): + expected_path = "pytest_files/test_snapshot0/updated.txt" + output_file = files / expected_path + res = pytest_command.run().ok() + assert output_file.read_text() == "plush plush" + assert "the updated file can be found here" not in res.stdout_plain