tests/functional2: add framework for lang tests
This creates a framework similar to the old lang.sh from functional. Some notable changes: - instead of having a .flags file, a test.toml can declare flags - additionally the test.toml can also declare extra files and multiple runners for the given input file. - there won't be any old tests hanging around anymore which weren't deleted properly in the installation - all files for a single test are defined decleratively and there won't be any residues Tests can be placed within the functional2/lang folder most migrations should be rather clean Implements: #825 Change-Id: I5f9149903ec5b078008969a4ae77305417c11475
This commit is contained in:
Commentator2.0
+9
-1
@@ -52,11 +52,17 @@ pre-commit-run {
|
||||
"\\.drv$"
|
||||
"^tests/functional/lang/"
|
||||
''\.patch$''
|
||||
"^tests/functional2/.+\\.exp"
|
||||
"^tests/functional2/.+\\.nix"
|
||||
];
|
||||
};
|
||||
mixed-line-endings = {
|
||||
enable = true;
|
||||
excludes = [ "^tests/functional/lang/" ];
|
||||
excludes = [
|
||||
"^tests/functional/lang/"
|
||||
"^tests/functional2/.+\\.exp"
|
||||
"^tests/functional2/.+\\.nix"
|
||||
];
|
||||
};
|
||||
release-notes = {
|
||||
enable = true;
|
||||
@@ -110,6 +116,8 @@ pre-commit-run {
|
||||
stages = [ "pre-commit" ];
|
||||
excludes = [
|
||||
''^tests/functional/lang/''
|
||||
"^tests/functional2/.+\\.exp"
|
||||
"^tests/functional2/.+\\.nix"
|
||||
''\.patch$''
|
||||
];
|
||||
};
|
||||
|
||||
@@ -200,6 +200,7 @@ let
|
||||
p.ruff
|
||||
p.python-frontmatter
|
||||
p.aiohttp
|
||||
p.toml
|
||||
]);
|
||||
in
|
||||
assert (lintInsteadOfBuild -> lix-clang-tidy != null);
|
||||
@@ -541,6 +542,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
p.ruff
|
||||
p.aiohttp
|
||||
p.python-frontmatter
|
||||
p.toml
|
||||
|
||||
p.yapf
|
||||
p.requests
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
error: syntax error, unexpected end of file, expecting '}'
|
||||
at /pwd/in.nix:1:2:
|
||||
1| {
|
||||
| ^
|
||||
@@ -0,0 +1 @@
|
||||
{ }
|
||||
@@ -0,0 +1,4 @@
|
||||
error: syntax error, unexpected end of file, expecting '}'
|
||||
at /pwd/in.nix:1:2:
|
||||
1| {
|
||||
| ^
|
||||
@@ -0,0 +1,2 @@
|
||||
_type: ExprSet
|
||||
recursive: false
|
||||
@@ -0,0 +1,305 @@
|
||||
import logging
|
||||
import re
|
||||
from enum import StrEnum
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import toml
|
||||
from toml import TomlDecodeError
|
||||
|
||||
from functional2.testlib.fixtures.file_helper import FileDeclaration, CopyFile, Symlink, RelativeTo
|
||||
from functional2.testlib.utils import test_base_folder
|
||||
|
||||
|
||||
LANG_TEST_ID_PATTERN = "{folder_name}:{test_name}"
|
||||
|
||||
|
||||
class LangTestRunner(StrEnum):
|
||||
"""
|
||||
A list of possible runners for the lang tests on
|
||||
The string value is the one present in the filenames / test.toml file as runner
|
||||
"""
|
||||
|
||||
EVAL_OKAY = "eval-okay"
|
||||
EVAL_FAIL = "eval-fail"
|
||||
PARSE_OKAY = "parse-okay"
|
||||
PARSE_FAIL = "parse-fail"
|
||||
|
||||
@classmethod
|
||||
def as_regex_selector(cls) -> str:
|
||||
"""
|
||||
Returns a regex to match any available runner name into a named capturing group called "runner_name"
|
||||
:return: a string containing a matching regex
|
||||
"""
|
||||
# For some reason ruff doesn't detect that .value is the string already and not a function
|
||||
# it seems to work when accessing `LangTestRunner` but not when using `cls`
|
||||
return rf"(?P<runner_name>{'|'.join([runner.value for runner in cls])})" # type: ignore
|
||||
|
||||
|
||||
INVALID_TESTER_NAME = (
|
||||
f"invalid runner name: '%s', must be one of {[runner.value for runner in LangTestRunner]!r}"
|
||||
)
|
||||
"""
|
||||
Base message for invalid runner, to use across collection
|
||||
"""
|
||||
|
||||
SUFFIX_REGEX = re.compile("-[\\w-]+?").pattern
|
||||
NAMING_PATTERN_LANG_TEST = re.compile(
|
||||
rf"{LangTestRunner.as_regex_selector()}(?P<suffix>{SUFFIX_REGEX})?"
|
||||
)
|
||||
|
||||
|
||||
class LangTest:
|
||||
def __init__(
|
||||
self,
|
||||
test_name: str,
|
||||
folder_name: str,
|
||||
runner: LangTestRunner,
|
||||
flags: list[str] | None = None,
|
||||
extra_files: list[str] | None = None,
|
||||
suffix: str = "",
|
||||
):
|
||||
"""
|
||||
Internal class to represent a lang test
|
||||
:param test_name: name of the explicit test (e.g. "eval-depr" or "eval-allow-depr")
|
||||
:param folder_name: the folder / group of tests this one originates from (e.g. "nul_bytes")
|
||||
:param runner: which runner to run this test on (e.g. EVAL_FAIL or PARSE_OKAY)
|
||||
:param flags: additional flags provided for nix
|
||||
:param extra_files: any additional files which should be copied into the tests directory
|
||||
:param suffix: suffix of the in file
|
||||
"""
|
||||
self.test_name = test_name
|
||||
self.full_name = LANG_TEST_ID_PATTERN.format(
|
||||
folder_name=folder_name, test_name=f"{test_name}{suffix}"
|
||||
)
|
||||
self.runner = runner
|
||||
self.flags = flags or []
|
||||
self.folder = folder_name
|
||||
self.extra_files = extra_files or []
|
||||
self.suffix = suffix
|
||||
|
||||
def _get_files(self) -> FileDeclaration:
|
||||
"""
|
||||
Internal function to turn the metadata into a FileDeclaration used for parametrization
|
||||
:return: FileDeclaration object containing all files required for the test
|
||||
"""
|
||||
files = {
|
||||
"in.nix": CopyFile(f"{self.folder}/in{self.suffix}.nix"),
|
||||
"lib.nix": CopyFile("lib.nix"),
|
||||
"out.exp": Symlink(
|
||||
f"{self.folder}/{self.test_name}{self.suffix}.out.exp", relative_to=RelativeTo.TEST
|
||||
),
|
||||
"err.exp": Symlink(
|
||||
f"{self.folder}/{self.test_name}{self.suffix}.err.exp", relative_to=RelativeTo.TEST
|
||||
),
|
||||
}
|
||||
for file in self.extra_files:
|
||||
# Make sure to add the extra-files requested by the test.toml
|
||||
files[file] = CopyFile(f"{self.folder}/{file}")
|
||||
return files
|
||||
|
||||
def to_params(self) -> tuple[FileDeclaration, list[str], str]:
|
||||
"""
|
||||
Converts the LangTest to the parameters required for parametrization of the test runners
|
||||
:return: a Tuple of the FileDeclaration (used by the `files` fixture), list of flags and a unique id
|
||||
"""
|
||||
return self._get_files(), self.flags, self.full_name
|
||||
|
||||
|
||||
class InvalidLangTest:
|
||||
def __init__(self, name: str, reasons: list[str]):
|
||||
"""
|
||||
Metadata class for invalid test configuration, used to later fail a test with the given reasons
|
||||
:param name: name of the test which is configured badly
|
||||
:param reasons: a list of reasons as to why the configuration is invalid
|
||||
"""
|
||||
self.name = name
|
||||
self.reasons = reasons
|
||||
|
||||
|
||||
def _group_lang_tests(tests: list[LangTest]) -> dict[LangTestRunner, list[LangTest]]:
|
||||
"""
|
||||
groups the given list of tests by their runner
|
||||
:param tests: list of tests to group
|
||||
:return: grouped tests in the following order: EVAL_OKAY, EVAL_FAIL, PARSE_OKAY, PARSE_FAIL
|
||||
"""
|
||||
grouped_tests = {runner_name: [] for runner_name in LangTestRunner}
|
||||
for test in tests:
|
||||
grouped_tests[test.runner].append(test)
|
||||
return grouped_tests
|
||||
|
||||
|
||||
def _is_list_of_type(value: Any, expected_type: type[Any]) -> bool:
|
||||
"""
|
||||
checks if the given value conforms to the type `list[expected_type]`
|
||||
:param value: value to check
|
||||
:param expected_type: what type each item should be
|
||||
:return: True, if it conforms, False otherwise
|
||||
"""
|
||||
return isinstance(value, list) and all(isinstance(v, expected_type) for v in value)
|
||||
|
||||
|
||||
def _is_list_of_strings(value: Any) -> bool:
|
||||
"""
|
||||
same as `_is_list_of_type` but with type `str` pre-applied
|
||||
:param value: value to check
|
||||
:return: True, if the value conforms to `list[str]` otherwise False
|
||||
"""
|
||||
return _is_list_of_type(value, str)
|
||||
|
||||
|
||||
def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
|
||||
"""
|
||||
Collects all tests, declared by a `test.toml` file within the given folder
|
||||
:param folder: base folder to collect tests in
|
||||
:return: a list of valid test configurations and a list of invalid test configurations
|
||||
"""
|
||||
parent_name = folder.name
|
||||
test_declaration = folder / "test.toml"
|
||||
try:
|
||||
infos: dict[str, Any] = toml.load(test_declaration)
|
||||
except TomlDecodeError as e:
|
||||
return [], [InvalidLangTest(parent_name, [f"couldn't parse toml: {e!r}"])]
|
||||
invalid_tests: list[InvalidLangTest] = []
|
||||
tests: list[LangTest] = []
|
||||
|
||||
# suffixes of the in files, e.g.
|
||||
# in.nix => ''
|
||||
# in-1.nix => '-1'
|
||||
# in-some-test.nix => '-some-test'
|
||||
# etc
|
||||
in_suffixes = [
|
||||
suffix.group(1) or ""
|
||||
for suffix in [
|
||||
re.fullmatch(rf"in({SUFFIX_REGEX})?\.nix", file.name) for file in folder.iterdir()
|
||||
]
|
||||
if suffix is not None
|
||||
]
|
||||
|
||||
for test_name, definition in infos.items():
|
||||
test_errors: list[str] = []
|
||||
full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name)
|
||||
if not isinstance(definition, dict):
|
||||
invalid_tests.append(
|
||||
InvalidLangTest(
|
||||
full_name, [f"invalid value for {test_name!r}; only tests are expected"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
flags = definition.pop("flags", [])
|
||||
if not _is_list_of_strings(flags):
|
||||
test_errors.append(
|
||||
f"invalid value type for 'flags': {flags}, expected a list of strings"
|
||||
)
|
||||
|
||||
runner_name = definition.pop("runner", None)
|
||||
try:
|
||||
runner = LangTestRunner(runner_name)
|
||||
except ValueError:
|
||||
test_errors.append(INVALID_TESTER_NAME % runner_name)
|
||||
runner = None
|
||||
|
||||
extra_files = definition.pop("extra-files", [])
|
||||
if not _is_list_of_strings(extra_files):
|
||||
test_errors.append(
|
||||
f"invalid value type for 'extra_files': {extra_files}, expected a list of strings"
|
||||
)
|
||||
|
||||
if len(definition) > 0:
|
||||
test_errors.append(f"unexpected arguments: {list(definition.keys())!r}")
|
||||
if test_errors:
|
||||
invalid_tests.append(InvalidLangTest(full_name, test_errors))
|
||||
continue
|
||||
|
||||
# Add a test for each in file
|
||||
tests += [
|
||||
LangTest(test_name, folder.name, runner, flags, extra_files, in_suffix)
|
||||
for in_suffix in in_suffixes
|
||||
]
|
||||
|
||||
return tests, invalid_tests
|
||||
|
||||
|
||||
def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
|
||||
"""
|
||||
Collects all tests within the given folder, if no `test.toml` file is presented for a more explicit test configuration
|
||||
:param folder: base folder to collect tests in
|
||||
:return: a list of valid test configurations and a list of invalid test configurations
|
||||
"""
|
||||
parent_name = folder.name
|
||||
tests: list[LangTest] = []
|
||||
invalid_tests: list[InvalidLangTest] = []
|
||||
for file in folder.iterdir():
|
||||
file: Path
|
||||
if file.suffix == ".exp":
|
||||
# we cannot use `file.stem` here, as it only removes the last suffix. i.e.
|
||||
# `"eval-okay.out.exp".stem` => "eval-okay.out"
|
||||
# `"eval-okay.out.exp".split(".")[0]` => "eval-okay"
|
||||
# or
|
||||
# `"parse-fail-some-name.err.exp".stem` => "parse-fail-some-name.err"
|
||||
# `"parse-fail-some-name.err.exp".split(".")[0]` => "parse-fail-some-name"
|
||||
test_name = file.name.rsplit(".", 2)[0]
|
||||
full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name)
|
||||
match = re.fullmatch(NAMING_PATTERN_LANG_TEST, test_name)
|
||||
if match is None:
|
||||
if re.match(LangTestRunner.as_regex_selector(), test_name) is None:
|
||||
reason = INVALID_TESTER_NAME % test_name
|
||||
else:
|
||||
reason = f"incorrectly formatted test name: {test_name!r}"
|
||||
invalid_tests.append(InvalidLangTest(full_name, [reason]))
|
||||
continue
|
||||
runner_name, suffix = match.groups()
|
||||
runner = LangTestRunner(runner_name)
|
||||
tests.append(LangTest(runner_name, folder.name, runner, suffix=suffix or ""))
|
||||
return tests, invalid_tests
|
||||
|
||||
|
||||
def _collect_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
|
||||
"""
|
||||
Collects all tests within the given test group
|
||||
:param folder: where the test group are located
|
||||
:return: a list of valid test configurations and a list of invalid test configurations
|
||||
"""
|
||||
test_declaration = folder / "test.toml"
|
||||
if test_declaration.exists():
|
||||
return _collect_toml_test_group(folder)
|
||||
return _collect_generic_test_group(folder)
|
||||
|
||||
|
||||
def _collect_all_tests() -> tuple[list[LangTest], list[InvalidLangTest]]:
|
||||
"""
|
||||
Collects all LangTests present in the functional2/lang folder
|
||||
:return: a list of valid test configurations and a list of invalid test configurations
|
||||
"""
|
||||
logger = logging.getLogger("lang-test-collector")
|
||||
lang_folder = test_base_folder / "functional2/lang"
|
||||
tests: list[LangTest] = []
|
||||
invalid_tests: list[InvalidLangTest] = []
|
||||
for node in lang_folder.iterdir():
|
||||
node: Path
|
||||
# skip files, as test groups are folders and custom tests will be collected by pytest
|
||||
# these are files like this `lang_util.py` or the `lib.nix` etc
|
||||
if node.is_file() or node.name == "assets":
|
||||
continue
|
||||
|
||||
# ignore test groups, which have a py file, as those are set up fully custom
|
||||
# and expected to be collected by pytest and not by us
|
||||
if len(list(node.glob("*.py"))):
|
||||
logger.info("skipping %s as it contains a py file, assuming custom tests", node)
|
||||
continue
|
||||
t, i = _collect_test_group(node)
|
||||
tests += t
|
||||
invalid_tests += i
|
||||
return tests, invalid_tests
|
||||
|
||||
|
||||
@cache
|
||||
def fetch_all_lang_tests() -> tuple[dict[LangTestRunner, list[LangTest]], list[InvalidLangTest]]:
|
||||
"""
|
||||
Collects all LangTests declared within `functional2/lang` and groups them by runner / type
|
||||
:return: valid tests as a mapping of runner -> tests for said runner, InvalidLangTests
|
||||
"""
|
||||
tests, invalid_tests = _collect_all_tests()
|
||||
return _group_lang_tests(tests), invalid_tests
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../functional/lang/lib.nix
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from _pytest.fixtures import FixtureRequest
|
||||
from _pytest.python import Metafunc
|
||||
|
||||
from functional2.lang.lang_util import LangTest, fetch_all_lang_tests, LangTestRunner
|
||||
from functional2.testlib.fixtures.nix import Nix
|
||||
from functional2.testlib.fixtures.snapshot import Snapshot
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: Metafunc):
|
||||
"""
|
||||
This hook parametrizes the parser and eval tests with all test cases found in functional2/lang
|
||||
:param metafunc: the test function to parametrize, provided by pytest
|
||||
"""
|
||||
func_name = metafunc.function.__name__
|
||||
tests, invalid_tests = fetch_all_lang_tests()
|
||||
if func_name == "test_invalid_configuration":
|
||||
metafunc.parametrize(("name", "reasons"), [(t.name, t.reasons) for t in invalid_tests])
|
||||
return
|
||||
selected_runner: LangTestRunner
|
||||
match func_name:
|
||||
case "test_eval":
|
||||
selected_runner = LangTestRunner.EVAL_OKAY
|
||||
case "test_xfail_eval":
|
||||
selected_runner = LangTestRunner.EVAL_FAIL
|
||||
case "test_parser":
|
||||
selected_runner = LangTestRunner.PARSE_OKAY
|
||||
case "test_xfail_parser":
|
||||
selected_runner = LangTestRunner.PARSE_FAIL
|
||||
case _:
|
||||
return
|
||||
selected_tests = tests[selected_runner]
|
||||
if len(selected_tests) > 0:
|
||||
# ignoring type here, because map doesn't recognize the typing correctly
|
||||
# due to returning multiple things and it expecting a single generic
|
||||
files, flags, ids = zip(*map(LangTest.to_params, selected_tests)) # type: ignore
|
||||
else:
|
||||
files, flags, ids = [], [], []
|
||||
metafunc.parametrize(("files", "flags"), zip(files, flags), ids=ids, indirect=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def flags(request: FixtureRequest) -> list[str]:
|
||||
return request.param
|
||||
|
||||
|
||||
def _cleanup_output(stdout: str, stderr: str, origin: Path) -> tuple[str, str]:
|
||||
"""
|
||||
Cleans up any paths present in stdout and stderr by replacing them with placeholders
|
||||
"""
|
||||
test_path = str(origin)
|
||||
clean_out = stdout.replace(test_path, "/pwd")
|
||||
clean_err = stderr.replace(test_path, "/pwd")
|
||||
return clean_out, clean_err
|
||||
|
||||
|
||||
def test_parser(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]):
|
||||
nix_command = nix.nix_instantiate(
|
||||
["--parse", *flags, files / "in.nix"],
|
||||
# TODO(Commentator2.0): Mirrors behavior of init.sh from functional
|
||||
# keep this for migration, but make it declarative afterwards
|
||||
# and only active for the tests which need it
|
||||
flake=True,
|
||||
)
|
||||
result = nix_command.run().ok()
|
||||
stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files)
|
||||
|
||||
# Taken from https://stackoverflow.com/a/39681672
|
||||
# Pyyaml does not give list items extra indentation, unlike pretty much everyone else.
|
||||
# For migration compatibility with the `yq` output from the functional/lang tests, we
|
||||
# use a custom dumper that produces the correct indentation.
|
||||
# Upstream issue (open since 2018): https://github.com/yaml/pyyaml/issues/234
|
||||
class CustomFixedIndentationDumper(yaml.Dumper):
|
||||
def increase_indent(self, flow: bool = False, indentless: bool = False): # noqa: ARG002
|
||||
super().increase_indent(flow, False)
|
||||
|
||||
result_obj = json.loads(stdout)
|
||||
result_yaml = yaml.dump(
|
||||
result_obj, Dumper=CustomFixedIndentationDumper, default_flow_style=False
|
||||
)
|
||||
# parser out are in a yaml format, which is why we convert it here for better viewing and editing
|
||||
assert snapshot("out.exp") == result_yaml
|
||||
assert snapshot("err.exp") == stderr
|
||||
|
||||
|
||||
def test_xfail_parser(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]):
|
||||
nix_command = nix.nix_instantiate(["--parse", *flags, files / "in.nix"], flake=True)
|
||||
result = nix_command.run().expect(1)
|
||||
stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files)
|
||||
|
||||
assert snapshot("out.exp") == stdout
|
||||
assert snapshot("err.exp") == stderr
|
||||
|
||||
|
||||
def test_eval(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]):
|
||||
nix_command = nix.nix_instantiate(["--eval", "--strict", *flags, files / "in.nix"], flake=True)
|
||||
result = nix_command.run().ok()
|
||||
stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files)
|
||||
|
||||
assert snapshot("out.exp") == stdout
|
||||
assert snapshot("err.exp") == stderr
|
||||
|
||||
|
||||
def test_xfail_eval(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]):
|
||||
nix_command = nix.nix_instantiate(
|
||||
["--eval", "--strict", "--show-trace", *flags, files / "in.nix"], flake=True
|
||||
)
|
||||
result = nix_command.run().expect(1)
|
||||
stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files)
|
||||
|
||||
assert snapshot("out.exp") == stdout
|
||||
assert snapshot("err.exp") == stderr
|
||||
|
||||
|
||||
def test_invalid_configuration(name: str, reasons: list[str], logger: Logger):
|
||||
msg = f"Invalid configuration for {name!r}: {reasons}"
|
||||
logger.error(msg)
|
||||
pytest.fail(msg)
|
||||
@@ -0,0 +1,516 @@
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
from functional2.testlib.commands import Command
|
||||
from functional2.testlib.fixtures.file_helper import File, RelativeTo, Symlink
|
||||
from functional2.testlib.fixtures.snapshot import Snapshot
|
||||
from functional2.testlib.utils import get_functional2_lang_files
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"generic_test": {
|
||||
"in.nix": File("{}"),
|
||||
"eval-okay.out.exp": File("{ }\n"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "generic_test", "--setup-plan"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_detects_generic_lang_test(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
assert "lang/test_lang.py::test_eval[generic_test:eval-okay]" in result.stdout_plain
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"toml_test": {
|
||||
"in.nix": File("{}"),
|
||||
"my_name.out.exp": File("{ }\n"),
|
||||
"test.toml": File(
|
||||
dedent("""
|
||||
[my_name]
|
||||
runner = "eval-okay"
|
||||
""")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "toml_test", "--setup-plan"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_detects_toml_lang_test(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
assert "lang/test_lang.py::test_eval[toml_test:my_name]" in result.stdout_plain
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"some_py_module": {
|
||||
"in.nix": File("{}"),
|
||||
"eval-okay.out.exp": File("{ }\n"),
|
||||
"__init__.py": File(""),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["--setup-plan"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_skips_py_files(files: Path, pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
assert "lang/test_lang.py::test_eval[some_py_module:eval-okay]" not in result.stdout_plain
|
||||
assert (
|
||||
f"[ INFO] [lang-test-collector] skipping {files.absolute()}/functional2/lang/some_py_module as it contains a py file, assuming custom tests"
|
||||
in result.stdout_plain
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"n_suffix": {
|
||||
"in-1.nix": File("{}"),
|
||||
"in-2.nix": File("{}"),
|
||||
"eval-okay-1.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"eval-okay-2.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "n_suffix"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_collects_with_n_suffix(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
out = result.stdout_plain
|
||||
assert "n_suffix:eval-okay-1" in out
|
||||
assert "n_suffix:eval-okay-2" in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"short_string_suffix": {
|
||||
"in-speaker.nix": File("{}"),
|
||||
"in-microphone.nix": File("{}"),
|
||||
"eval-okay-speaker.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"eval-okay-microphone.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "short_string_suffix"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_collects_short_string_suffix(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
out = result.stdout_plain
|
||||
assert "short_string_suffix:eval-okay-speaker" in out
|
||||
assert "short_string_suffix:eval-okay-microphone" in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"dash_suffix": {
|
||||
"in-string-with-dash.nix": File("{}"),
|
||||
"in-some-more--dashes.nix": File("{}"),
|
||||
"eval-okay-string-with-dash.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"eval-okay-some-more--dashes.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "dash_suffix"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_collects_string_suffix_with_dash(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
out = result.stdout_plain
|
||||
assert "dash_suffix:eval-okay-string-with-dash" in out
|
||||
assert "dash_suffix:eval-okay-some-more--dashes" in out
|
||||
assert "dash_suffix:eval-okay-" in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"bad_naming": {
|
||||
"in-&x.nix": File("{}"),
|
||||
"in-.nix": File("{}"),
|
||||
"eval-okay-&x.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"eval-okay-.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "bad_naming"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_collection_fails_with_bad_naming(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_plain
|
||||
assert "bad_naming:eval-okay" in err
|
||||
assert "incorrectly formatted test name: 'eval-okay-'" in err
|
||||
assert "incorrectly formatted test name: 'eval-okay-&x'" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"infra_okay_runners": {
|
||||
"in.nix": File("{}"),
|
||||
"eval-okay.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"parse-okay.out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_po.out.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
},
|
||||
"infra_fail_runners": {
|
||||
"in.nix": File("{"),
|
||||
"eval-fail.err.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_ef.err.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
"parse-fail.err.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_pf.err.exp",
|
||||
relative_to=RelativeTo.TEST,
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "infra and runners"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_all_runners_work(pytest_command: Command):
|
||||
result = pytest_command.run().ok()
|
||||
assert "lang/test_lang.py::test_eval[infra_okay_runners:eval-okay]" in result.stdout_plain
|
||||
assert "lang/test_lang.py::test_parser[infra_okay_runners:parse-okay]" in result.stdout_plain
|
||||
assert "lang/test_lang.py::test_xfail_eval[infra_fail_runners:eval-fail]" in result.stdout_plain
|
||||
assert (
|
||||
"lang/test_lang.py::test_xfail_parser[infra_fail_runners:parse-fail]" in result.stdout_plain
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"generic_bad": {
|
||||
"in.nix": File("{}"),
|
||||
"fee-foo.out.exp": File(""),
|
||||
"hello-okay.out.exp": File(""),
|
||||
"parse-fops.out.exp": File(""),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
[],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_generic_bad_runner_name(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_s
|
||||
assert "test_invalid_configuration" in err
|
||||
assert "invalid runner name:" in err
|
||||
assert "Invalid configuration for 'generic_bad:fee-foo'" in err
|
||||
assert "Invalid configuration for 'generic_bad:hello-okay'" in err
|
||||
assert "Invalid configuration for 'generic_bad:parse-fops'" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"toml_test": {
|
||||
"in.nix": File("{}"),
|
||||
"my_name.out.exp": File("{ }\n"),
|
||||
"test.toml": File(
|
||||
dedent("""
|
||||
[my_name]
|
||||
runner = "plushies"
|
||||
""")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "toml_test"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_toml_bad_runner_name(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_plain
|
||||
assert "test_invalid_configuration" in err
|
||||
assert "invalid runner name:" in err
|
||||
assert "Invalid configuration for 'toml_test:my_name':" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"toml_test": {
|
||||
"in.nix": File("{}"),
|
||||
"my_name.out.exp": File("{ }\n"),
|
||||
"test.toml": File(
|
||||
dedent("""
|
||||
[my_name]
|
||||
runner = "eval-okay"
|
||||
cuddles = true
|
||||
""")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "toml_test"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_toml_too_many_args(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_plain
|
||||
assert "test_invalid_configuration" in err
|
||||
assert "unexpected arguments: ['cuddles']" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"toml_test": {
|
||||
"in.nix": File("{}"),
|
||||
"my_name.out.exp": File("{ }\n"),
|
||||
"test.toml": File(
|
||||
dedent("""
|
||||
invalid_test = "eval-okay"
|
||||
[my_name]
|
||||
runner = "eval-okay"
|
||||
flags = 1
|
||||
[second]
|
||||
runner = "eval-okay"
|
||||
extra-files = [false, true, true]
|
||||
""")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "toml_test"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_toml_invalid_argument_types(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_plain
|
||||
assert "test_invalid_configuration" in err
|
||||
assert "invalid value for 'invalid_test'; only tests are expected" in err
|
||||
assert "invalid value type for 'flags'" in err
|
||||
assert "invalid value type for 'extra_files':" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"toml_test": {
|
||||
"in.nix": File("{}"),
|
||||
"my_name.out.exp": File("{ }\n"),
|
||||
"test.toml": File(
|
||||
dedent("""
|
||||
[my_name]
|
||||
runner = "eval-okay"
|
||||
cuddles = True
|
||||
""")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
["-k", "toml_test"],
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.usefixtures("files")
|
||||
def test_invalid_toml(pytest_command: Command):
|
||||
result = pytest_command.run().expect(1)
|
||||
err = result.stdout_plain
|
||||
assert "test_invalid_configuration" in err
|
||||
assert "couldn't parse toml" in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "pytest_command"),
|
||||
[
|
||||
(
|
||||
get_functional2_lang_files(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"update_test": {"in.nix": File("{}"), "eval-okay.out.exp": File("old")}
|
||||
}
|
||||
},
|
||||
"out.exp": Symlink(
|
||||
"assets/test_lang_infra/runner_eo.out.exp", relative_to=RelativeTo.TEST
|
||||
),
|
||||
}
|
||||
),
|
||||
(["-k", "update_test", "--accept-tests"], False),
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_updates_expected_output(
|
||||
files: Path, pytest_command: Command, snapshot: Callable[[str], Snapshot]
|
||||
):
|
||||
assert (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() == "old"
|
||||
pytest_command.run().ok()
|
||||
assert (
|
||||
snapshot("out.exp")
|
||||
== (files / "functional2/lang/update_test/eval-okay.out.exp").read_text()
|
||||
)
|
||||
@@ -1,8 +1,12 @@
|
||||
xdist_opts = [
|
||||
# auto number of workers, max 12 jobs
|
||||
'-n', 'auto', '--maxprocesses=12',
|
||||
# group tests by module or class; ensures that any setup work occurs as little as possible
|
||||
'--dist=loadscope',
|
||||
# Distributes tests evenly at first and then, if workers run out of tests, they steal from a different worker
|
||||
# This is useful here, due to differing durations of tests, especially when tests check timeout functionality
|
||||
#
|
||||
# grouping by module/file is disadventageous here too, as all lang tests are executed from the same module
|
||||
# which would result in high run times for that single worker while everything else is bored
|
||||
'--dist=worksteal',
|
||||
]
|
||||
|
||||
# surprisingly, this actually works even if PATH is set to something before
|
||||
@@ -16,6 +20,8 @@ test(
|
||||
args : [
|
||||
'-m', 'pytest',
|
||||
'-v',
|
||||
# Print out summaries of **f**ailed, **E**rrored and **s**kipped tests
|
||||
'-r', 'fEs',
|
||||
xdist_opts,
|
||||
meson.current_source_dir()
|
||||
],
|
||||
|
||||
@@ -66,3 +66,23 @@ def get_functional2_files_with_testlib(
|
||||
additional_files,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_functional2_lang_files(additional_files: FileDeclaration | None = None) -> FileDeclaration:
|
||||
if additional_files is None:
|
||||
additional_files = {}
|
||||
return get_functional2_files_with_testlib(
|
||||
merge_file_declaration(
|
||||
{
|
||||
"functional2": {
|
||||
"lang": {
|
||||
"__init__.py": CopyFile(functional2_base_folder / "lang/__init__.py"),
|
||||
"lang_util.py": CopyFile(functional2_base_folder / "lang/lang_util.py"),
|
||||
"test_lang.py": CopyFile(functional2_base_folder / "lang/test_lang.py"),
|
||||
"lib.nix": CopyFile(functional2_base_folder / "lang/lib.nix"),
|
||||
}
|
||||
}
|
||||
},
|
||||
additional_files,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user