tests/functional2: improve files ux

Currenlty one is required to always write the bulky `mark.parametrize`
with indirect and things

This commit adds a custom decorator for usage of files, which hides the
parametrization complexity from the user.

Change-Id: I526e016d12006669dc302dfc5af619735399c503
This commit is contained in:
Commentator2.0
2025-08-22 07:50:20 +02:00
committed by Commentator2.0
parent a84355a813
commit 0a3e43590c
7 changed files with 604 additions and 846 deletions
+36 -27
View File
@@ -23,7 +23,6 @@ Tests for the test suite itself are located in the `testlib` package.
```just test functional2``` will run the entire test suite which will call Python from our meson build infrastructure.
Alternatively, ```just test-functional2``` will run the test suite through pytest directly, which allows providing additional arguments to pytest.
Additionally, ```just test-functional2-parallel``` will do the same but in parallel, trading faster run time against a larger start-up time and a lack of detailed log output.
A quick primer on useful `pytest` arguments:
@@ -126,26 +125,24 @@ def test_both_params_at_once(a: int, b: str):
Injecting parameters into a test that are controlled by [fixtures](#useful-fixtures) requires setting `indirect=True` to the parametrization:
This is currently only used by the `pytest_command` fixture, to test our framework.
```python
import pytest
from pathlib import Path
from functional2.testlib.fixtures.file_helper import File
@pytest.mark.parametrize("files",
from functional2.testlib.fixtures.pytest_command import Command
@pytest.mark.parametrize("pytest_command",
[
{"test.txt": File("first fileset")},
{"test.txt": File("second fileset")},
["-k", "fun"],
["-k", "cake", "--accept-tests"],
],
indirect=True,
)
def test_using_files(files: Path):
def test_pytest_collection(pytest_command: Command):
# is called twice, resulting in the following output:
# first fileset
# second fileset
print((files / "test.txt").read_text())
# ["pytest", "-k", "fun"]
# ["pytest", "-k", "cake", "--accept-tests"]
print(pytest_command.argv)
```
Without `indirect=True` here, the `test_using_files` would be called with the raw dictionary values (`{"test.txt": File("first fileset")}` etc.) instead of the desired file paths that the fixture provides.
Without `indirect=True`, the function would get the raw argument lists and not the configured Command.
### Useful fixtures
@@ -186,19 +183,32 @@ The input type is [FileDeclaration](./testlib/fixtures/file_helper.py), a dict f
- `Symlink("target/path")`: Create a symlink with the specified target. Therefore, relative paths are relative to the symlink's location.
- `AssetSymlink("source/path)`: Create a symlink pointing to a local asset file within the test suite. Paths must be relative and will be resolved relative to the current Python file. The created symlink will be absolute.
In order to make its usage easier, one can also use this fixture by using our custom decorator.
```python
import pytest
from pathlib import Path
from functional2.testlib.fixtures.file_helper import File, Symlink
from functional2.testlib.fixtures.file_helper import File, Symlink, with_files
@pytest.mark.parametrize(
"files",
# Arguments to the fixture
[ {
@with_files(
{
"test.txt": File("File content"),
"some-symlink": Symlink("../in.nix"),
} ],
indirect=True, # Required
}
)
def test_using_files(files: Path):
print((files / "test.txt").read_text())
```
To run a function twice with a different set of files, one can pass multiple dictionaries to the mark:
```python
from pathlib import Path
from functional2.testlib.fixtures.file_helper import File, with_files
@with_files(
{"test.txt": File("fileset 1")},
{"test.txt": File("fileset 2")},
)
def test_using_files(files: Path):
print((files / "test.txt").read_text())
@@ -216,15 +226,14 @@ In order for the golden values to actually update within the code base (compared
```python
import pytest
from functional2.testlib.fixtures.file_helper import AssetSymlink
from functional2.testlib.fixtures.file_helper import AssetSymlink, with_files
@pytest.mark.parametrize(
"files",
# Set up the symlink so that the golden value will update
[ { "out": AssetSymlink("assets/test_example/out.exp"), } ],
indirect=True,
@with_files(
# Set up the symlink so that the golden value will update
{ "out": AssetSymlink("assets/test_example/out.exp"), }
)
def test_example(files, snapshot):
def test_example(snapshot):
# snapshot must be on the LHS of ==
assert snapshot("out") == "Hello World"
```
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -4,7 +4,7 @@ import sqlite3
import aiohttp.web as web
import pytest
from functional2.testlib.fixtures.file_helper import File
from functional2.testlib.fixtures.file_helper import File, with_files
from functional2.testlib.fixtures.http_server import http_server
from functional2.testlib.fixtures.nix import Nix
@@ -100,7 +100,7 @@ def nars_from_narinfo_cache(db_path: Path) -> list[dict[str, str | bool]]:
]
@pytest.mark.parametrize("files", [{"test-file": File("hello world")}], indirect=True)
@with_files({"test-file": File("hello world")})
def test_http_simple(nix: Nix, store: HTTPStore, files: Path):
test_file = files / "test-file"
result = nix.nix(cmd=["store", "add-file", test_file], flake=True).run()
@@ -177,12 +177,21 @@ def _init_files(files: FileDeclaration, tmp_path: Path, request: pytest.FixtureR
_init_files(definition, destination, request)
def with_files(*files: FileDeclaration) -> Callable[[Any], Callable[[Any], None]]:
def decorator(func: Callable[[Any], None]) -> Callable[[Any], None]:
return pytest.mark.usefixtures("files")(
pytest.mark.parametrize("files", files, indirect=True)(func)
)
return decorator
@pytest.fixture
def files(env: ManagedEnv, request: pytest.FixtureRequest) -> Path:
"""
Initializes the given files into the TempDir of the test.
This ensures all necessary files and only those are present
To use this add `@pytest.mark.parametrize("files", [list_of_your_files_to_test, more_files_to_test], indirect=True)` above your test.
To use this add `@with_files(list_of_your_files_to_test, more_files_to_test)` above your test.
The test is run once for each of the sets of files provided as the second argument.
Each Set of files should be of the :py:type:`FileDeclaration` type
@@ -191,5 +200,6 @@ def files(env: ManagedEnv, request: pytest.FixtureRequest) -> Path:
:return: Path to where the files were created
"""
home = env.dirs.home
_init_files(request.param, home, request)
if hasattr(request, "param"):
_init_files(request.param, home, request)
return home
@@ -7,7 +7,7 @@ import pytest
from _pytest.logging import LogCaptureFixture
from functional2.testlib.fixtures.command import Command
from functional2.testlib.fixtures.env import ManagedEnv
from functional2.testlib.fixtures.file_helper import File
from functional2.testlib.fixtures.file_helper import File, with_files
def test_command_valid_runs(command: Callable[[list[str]], Command]):
@@ -39,18 +39,13 @@ def test_command_expect_failure(env: ManagedEnv):
Command(_env=env, argv=["grep", "xxx"], stdin=b"").run().expect(1)
@pytest.mark.parametrize(
"files",
[
{
"script.sh": File(
"#!/bin/sh\necho forb\nexit 1", mode=stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
)
}
],
indirect=True,
@with_files(
{
"script.sh": File(
"#!/bin/sh\necho forb\nexit 1", mode=stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
)
}
)
@pytest.mark.usefixtures("files")
def test_command_ok_fails_on_bad_exit_code(
command: Callable[[list[str]], Command], caplog: LogCaptureFixture
):
@@ -65,18 +60,13 @@ def test_command_ok_fails_on_bad_exit_code(
assert err_msg == "stderr: "
@pytest.mark.parametrize(
"files",
[
{
"script.sh": File(
"#!/bin/sh\necho drgn fops\nexit 2", mode=stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
)
}
],
indirect=True,
@with_files(
{
"script.sh": File(
"#!/bin/sh\necho drgn fops\nexit 2", mode=stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
)
}
)
@pytest.mark.usefixtures("files")
def test_command_exec_fails_on_other_bad_exit_code(
command: Callable[[list[str]], Command], caplog: LogCaptureFixture
):
@@ -10,12 +10,11 @@ from functional2.testlib.fixtures.file_helper import (
Symlink,
AssetSymlink,
merge_file_declaration,
with_files,
)
@pytest.mark.parametrize(
"files", [{"test_file_1": File("this is some test content")}], indirect=True
)
@with_files({"test_file_1": File("this is some test content")})
def test_file_creates(files: Path):
file_name = "test_file_1"
file_content = "this is some test content"
@@ -25,7 +24,7 @@ def test_file_creates(files: Path):
assert target_path.read_text() == file_content
@pytest.mark.parametrize("files", [{"test_file_2": File("asdf")}], indirect=True)
@with_files({"test_file_2": File("asdf")})
def test_no_file_leakage(files: Path):
file_name = "test_file_2"
file_content = "asdf"
@@ -38,11 +37,7 @@ def test_no_file_leakage(files: Path):
assert not (files / some_other_file_name).exists()
@pytest.mark.parametrize(
"files",
[{"copy_file_test.txt": CopyFile("assets/test_file_helper/copy_file_test.txt")}],
indirect=True,
)
@with_files({"copy_file_test.txt": CopyFile("assets/test_file_helper/copy_file_test.txt")})
def test_copy_file_no_name(files: Path):
target_path = files / "copy_file_test.txt"
@@ -51,27 +46,19 @@ def test_copy_file_no_name(files: Path):
assert target_path.read_text() == "some amazing content\n"
@pytest.mark.parametrize(
"files",
[{"new_name.txt": CopyFile("assets/test_file_helper/copy_file_test.txt")}],
indirect=True,
)
@with_files({"new_name.txt": CopyFile("assets/test_file_helper/copy_file_test.txt")})
def test_copy_file_with_name(files: Path):
assert (files / "new_name.txt").exists()
assert not (files / "copy_file_test.txt").exists()
@pytest.mark.parametrize(
"files",
[
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{"some_key": "abc", "other_key": 123, "key_in_braces": "in_braces"},
)
}
],
indirect=True,
@with_files(
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{"some_key": "abc", "other_key": 123, "key_in_braces": "in_braces"},
)
}
)
def test_template(files: Path):
assert (files / "target.txt").exists()
@@ -88,17 +75,13 @@ def test_template(files: Path):
assert actual_content == expected_content
@pytest.mark.parametrize(
"files",
[
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{"some_key": "abc", "key_in_braces": "in_braces"},
)
}
],
indirect=True,
@with_files(
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{"some_key": "abc", "key_in_braces": "in_braces"},
)
}
)
@pytest.mark.xfail(raises=KeyError)
def test_template_missing_key(files: Path):
@@ -106,22 +89,18 @@ def test_template_missing_key(files: Path):
...
@pytest.mark.parametrize(
"files",
[
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{
"some_key": "Eragon",
"key_in_braces": "in_braces",
"other_key": "Arthur Leywin",
"the beginning": "after the end",
},
)
}
],
indirect=True,
@with_files(
{
"target.txt": CopyTemplate(
"assets/test_file_helper/copy_file_template.template",
{
"some_key": "Eragon",
"key_in_braces": "in_braces",
"other_key": "Arthur Leywin",
"the beginning": "after the end",
},
)
}
)
@pytest.mark.xfail(raises=ExceptionGroup)
def test_template_too_many_keys(files: Path):
@@ -129,9 +108,7 @@ def test_template_too_many_keys(files: Path):
...
@pytest.mark.parametrize(
"files", [{"some_folder": CopyTree("assets/test_file_helper/test_folder")}], indirect=True
)
@with_files({"some_folder": CopyTree("assets/test_file_helper/test_folder")})
def test_copy_tree(files: Path):
files = files / "some_folder"
assert (files / "a.txt").exists()
@@ -140,19 +117,15 @@ def test_copy_tree(files: Path):
assert (files / "sub_test" / "c.txt").exists()
@pytest.mark.parametrize(
"files",
[
{
"a.txt": File("Hello World"),
"assembler.txt": File("reinforced iron plates\n"),
"folder_1": {
"empty.nix": File(""),
"folder_2": {"not_empty.py": File("print('Arrruuuuuuuuuuuu')")},
},
}
],
indirect=True,
@with_files(
{
"a.txt": File("Hello World"),
"assembler.txt": File("reinforced iron plates\n"),
"folder_1": {
"empty.nix": File(""),
"folder_2": {"not_empty.py": File("print('Arrruuuuuuuuuuuu')")},
},
}
)
def test_files_sub_dirs(files: Path):
f = files / "a.txt"
@@ -174,25 +147,21 @@ def test_files_sub_dirs(files: Path):
assert f.exists()
@pytest.mark.parametrize(
"files", [{"not_empty": {"a.txt": File("Hello World")}, "empty": {}}], indirect=True
)
@with_files({"not_empty": {"a.txt": File("Hello World")}, "empty": {}})
def test_creates_empty_directory(files: Path):
assert (files / "not_empty").exists()
assert (files / "empty").exists()
assert (files / "empty").is_dir()
@pytest.mark.parametrize("files", [{"a.sh": File("echo test", mode=0o477)}], indirect=True)
@with_files({"a.sh": File("echo test", mode=0o477)})
def test_mode_setting(files: Path):
file = files / "a.sh"
assert file.exists()
assert file.stat().st_mode & 0o477 == 0o477
@pytest.mark.parametrize(
"files", [{"a": AssetSymlink("assets/test_file_helper/copy_file_test.txt")}], indirect=True
)
@with_files({"a": AssetSymlink("assets/test_file_helper/copy_file_test.txt")})
def test_asset_symlink(files: Path):
file = files / "a"
assert file.exists(follow_symlinks=False)
@@ -204,9 +173,7 @@ def test_asset_symlink(files: Path):
assert file.read_text() == "some amazing content\n"
@pytest.mark.parametrize(
"files", [{"a": AssetSymlink("assets/test_file_helper/test_folder")}], indirect=True
)
@with_files({"a": AssetSymlink("assets/test_file_helper/test_folder")})
def test_dir_symlink(files: Path):
folder = files / "a"
assert folder.exists(follow_symlinks=False)
@@ -220,9 +187,7 @@ def test_dir_symlink(files: Path):
assert (folder / "a.txt").exists()
@pytest.mark.parametrize(
"files", [{"a": AssetSymlink("assets/test_file_helper/this_does_not_exist")}], indirect=True
)
@with_files({"a": AssetSymlink("assets/test_file_helper/this_does_not_exist")})
def test_invalid_asset_symlink(files: Path):
link = files / "a"
assert link.exists(follow_symlinks=False)
@@ -231,18 +196,12 @@ def test_invalid_asset_symlink(files: Path):
@pytest.mark.xfail(raises=ValueError)
@pytest.mark.parametrize(
"files",
[{"a": AssetSymlink("/absolute/assets/test_file_helper/this_does_not_exist")}],
indirect=True,
)
@with_files({"a": AssetSymlink("/absolute/assets/test_file_helper/this_does_not_exist")})
def test_absolute_asset_symlink(files: Path):
pass
@pytest.mark.parametrize(
"files", [{"tg": File("Hello World"), "folder": {"link": Symlink("../tg")}}], indirect=True
)
@with_files({"tg": File("Hello World"), "folder": {"link": Symlink("../tg")}})
def test_file_symlink(files: Path):
assert (files / "tg").exists()
link = files / "folder" / "link"
@@ -252,6 +211,13 @@ def test_file_symlink(files: Path):
assert str(link.readlink()) == "../tg"
@with_files({"test-file.txt": File("a")}, {"test-file.txt": File("b")})
def test_multiple_file_sets(files: Path):
file = files / "test-file.txt"
assert file.exists()
assert file.read_text() == "a" or file.read_text() == "b"
def test_merge_fd_merges_correctly():
fa = File("a")
fc = File("c")
@@ -12,6 +12,7 @@ from functional2.testlib.fixtures.file_helper import (
File,
FileDeclaration,
merge_file_declaration,
with_files,
)
from functional2.testlib.fixtures.snapshot import Snapshot
from functional2.testlib.utils import get_functional2_files
@@ -45,15 +46,13 @@ def test_snapshot_empty_on_no_file(snapshot: Callable[[str], Snapshot]):
assert snapshot("this_file_does_not_exist") == "" # noqa: PLC1901
@pytest.mark.parametrize("files", [{"empty_file.txt": File("")}], indirect=True)
@pytest.mark.usefixtures("files")
@with_files({"empty_file.txt": File("")})
def test_snapshot_empty_on_empty_file(snapshot: Callable[[str], Snapshot]):
# no qa here, as we want to exactly check for empty string
assert snapshot("empty_file.txt") == "" # noqa: PLC1901
@pytest.mark.parametrize("files", [{"out.exp": File("Hell o' World")}], indirect=True)
@pytest.mark.usefixtures("files")
@with_files({"out.exp": File("Hell o' World")})
def test_snapshot_warns_on_non_symlink_file(
snapshot: Callable[[str], Snapshot], caplog: LogCaptureFixture
):
@@ -66,30 +65,23 @@ def test_snapshot_warns_on_non_symlink_file(
)
@pytest.mark.parametrize(
("files", "pytest_command"),
[
(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_noupdate(do_snapshot_update):
assert not do_snapshot_update
""")
)
}
}
@pytest.mark.parametrize("pytest_command", [([], False)], indirect=True)
@with_files(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_noupdate(do_snapshot_update):
assert not do_snapshot_update
""")
)
}
),
([], False),
)
],
indirect=True,
}
}
)
)
@pytest.mark.usefixtures("files")
def test_do_update_false_when_none_set(pytest_command: Command):
pytest_command.run().ok()
@@ -111,12 +103,10 @@ _update_test_files = _get_f2_snapshot_files(
@pytest.mark.parametrize(
("files", "pytest_command"),
[(_update_test_files, ([], False)), (_update_test_files, (["--accept-tests"], False))],
indirect=True,
"pytest_command", [([], False), (["--accept-tests"], False)], indirect=True
)
@pytest.mark.parametrize("set_env", [False, True])
@pytest.mark.usefixtures("files")
@with_files(_update_test_files)
def test_do_update_true_when_any_set(env: ManagedEnv, pytest_command: Command, set_env: bool):
if not (set_env or "--accept-tests" in pytest_command.argv):
pytest.skip("not this test case")
@@ -143,42 +133,30 @@ def _snapshot_test_files(content: str) -> FileDeclaration:
)
@pytest.mark.parametrize(
("files", "pytest_command"),
[(_snapshot_test_files("plush plush"), (["--accept-tests"], False))],
indirect=True,
)
@pytest.mark.usefixtures("files")
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(_snapshot_test_files("plush plush"))
def test_snapshot_no_updated_when_equal(pytest_command: Command):
res = pytest_command.run().ok()
assert "the updated file can be found here" not in res.stdout_plain
@pytest.mark.parametrize(
("files", "pytest_command"), [(_snapshot_test_files("fops plush"), ([], False))], indirect=True
)
@pytest.mark.usefixtures("files")
@pytest.mark.parametrize("pytest_command", [([], False)], indirect=True)
@with_files(_snapshot_test_files("fops plush"))
def test_snapshot_fails_on_diff(pytest_command: Command):
res = pytest_command.run().expect(1)
assert "FAILED test_snapshot/test_snapshot.py::test_snapshot" in res.stdout_plain
@pytest.mark.parametrize(
("files", "pytest_command"),
[(_snapshot_test_files("fops plush"), (["--accept-tests"], False))],
indirect=True,
)
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(_snapshot_test_files("fops plush"))
def test_snapshot_updates_diff(files: Path, pytest_command: Command):
output_file = files / "pytest_files/test_snapshot0/test-home/out.exp"
pytest_command.run().ok()
assert output_file.read_text() == "plush plush"
@pytest.mark.parametrize(
("files", "pytest_command"),
[(_snapshot_test_files("fops plush"), (["--accept-tests"], False))],
indirect=True,
)
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(_snapshot_test_files("fops plush"))
def test_snapshot_updates_shares_updated_location_without_symlink(
files: Path, pytest_command: Command
):
@@ -190,11 +168,8 @@ def test_snapshot_updates_shares_updated_location_without_symlink(
assert expected_path in res.stdout_plain
@pytest.mark.parametrize(
("files", "pytest_command"),
[(_snapshot_test_files("fops plush"), (["--accept-tests"], False))],
indirect=True,
)
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(_snapshot_test_files("fops plush"))
def test_snapshot_marks_skip_after_update(files: Path, pytest_command: Command):
expected_path = "pytest_files/test_snapshot0/test-home/out.exp"
output_file = files / expected_path
@@ -203,31 +178,25 @@ def test_snapshot_marks_skip_after_update(files: Path, pytest_command: Command):
assert "test_snapshot/test_snapshot.py::test_snapshot SKIPPED (Updated" in res.stdout_plain
@pytest.mark.parametrize(
("files", "pytest_command"),
[
(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_snapshot(snapshot, tmp_path):
(tmp_path / "test-home" / "out.exp").write_text("fops plush")
(tmp_path / "test-home" / "err.exp").write_text("shork plush")
assert snapshot("out.exp") == "plush plush"
assert snapshot("err.exp") == "blobhaj"
""")
)
}
}
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_snapshot(snapshot, tmp_path):
(tmp_path / "test-home" / "out.exp").write_text("fops plush")
(tmp_path / "test-home" / "err.exp").write_text("shork plush")
assert snapshot("out.exp") == "plush plush"
assert snapshot("err.exp") == "blobhaj"
""")
)
}
),
(["--accept-tests"], False),
)
],
indirect=True,
}
}
)
)
def test_snapshot_updates_multiple(files: Path, pytest_command: Command):
expected_path = "pytest_files/test_snapshot0/test-home"
@@ -239,30 +208,24 @@ def test_snapshot_updates_multiple(files: Path, pytest_command: Command):
assert second_file.read_text() == "blobhaj"
@pytest.mark.parametrize(
("files", "pytest_command"),
[
(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_snapshot(snapshot, tmp_path):
(tmp_path / "test-home" / "updated.txt").write_text("snek plush")
(tmp_path / "test-home" / "out.exp").symlink_to("./updated.txt")
assert snapshot("out.exp") == "plush plush"
""")
)
}
}
@pytest.mark.parametrize("pytest_command", [(["--accept-tests"], False)], indirect=True)
@with_files(
_get_f2_snapshot_files(
{
"functional2": {
"test_snapshot": {
"test_snapshot.py": File(
dedent("""
def test_snapshot(snapshot, tmp_path):
(tmp_path / "test-home" / "updated.txt").write_text("snek plush")
(tmp_path / "test-home" / "out.exp").symlink_to("./updated.txt")
assert snapshot("out.exp") == "plush plush"
""")
)
}
),
(["--accept-tests"], False),
)
],
indirect=True,
}
}
)
)
def test_snapshot_updates_no_location_when_symlink(files: Path, pytest_command: Command):
expected_path = "pytest_files/test_snapshot0/test-home/updated.txt"