diff --git a/misc/pre-commit.nix b/misc/pre-commit.nix index 17b344b73..3d0c551fe 100644 --- a/misc/pre-commit.nix +++ b/misc/pre-commit.nix @@ -52,11 +52,17 @@ pre-commit-run { "\\.drv$" "^tests/functional/lang/" ''\.patch$'' + "^tests/functional2/.+\\.exp" + "^tests/functional2/.+\\.nix" ]; }; mixed-line-endings = { enable = true; - excludes = [ "^tests/functional/lang/" ]; + excludes = [ + "^tests/functional/lang/" + "^tests/functional2/.+\\.exp" + "^tests/functional2/.+\\.nix" + ]; }; release-notes = { enable = true; @@ -110,6 +116,8 @@ pre-commit-run { stages = [ "pre-commit" ]; excludes = [ ''^tests/functional/lang/'' + "^tests/functional2/.+\\.exp" + "^tests/functional2/.+\\.nix" ''\.patch$'' ]; }; diff --git a/package.nix b/package.nix index 3e7c5889b..57dd07bdb 100644 --- a/package.nix +++ b/package.nix @@ -200,6 +200,7 @@ let p.ruff p.python-frontmatter p.aiohttp + p.toml ]); in assert (lintInsteadOfBuild -> lix-clang-tidy != null); @@ -541,6 +542,7 @@ stdenv.mkDerivation (finalAttrs: { p.ruff p.aiohttp p.python-frontmatter + p.toml p.yapf p.requests diff --git a/tests/functional2/lang/__init__.py b/tests/functional2/lang/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional2/lang/assets/test_lang_infra/runner_ef.err.exp b/tests/functional2/lang/assets/test_lang_infra/runner_ef.err.exp new file mode 100644 index 000000000..e322e0b44 --- /dev/null +++ b/tests/functional2/lang/assets/test_lang_infra/runner_ef.err.exp @@ -0,0 +1,4 @@ +error: syntax error, unexpected end of file, expecting '}' + at /pwd/in.nix:1:2: + 1| { + | ^ diff --git a/tests/functional2/lang/assets/test_lang_infra/runner_eo.out.exp b/tests/functional2/lang/assets/test_lang_infra/runner_eo.out.exp new file mode 100644 index 000000000..ffcd4415b --- /dev/null +++ b/tests/functional2/lang/assets/test_lang_infra/runner_eo.out.exp @@ -0,0 +1 @@ +{ } diff --git a/tests/functional2/lang/assets/test_lang_infra/runner_pf.err.exp b/tests/functional2/lang/assets/test_lang_infra/runner_pf.err.exp new file mode 100644 index 000000000..e322e0b44 --- /dev/null +++ b/tests/functional2/lang/assets/test_lang_infra/runner_pf.err.exp @@ -0,0 +1,4 @@ +error: syntax error, unexpected end of file, expecting '}' + at /pwd/in.nix:1:2: + 1| { + | ^ diff --git a/tests/functional2/lang/assets/test_lang_infra/runner_po.out.exp b/tests/functional2/lang/assets/test_lang_infra/runner_po.out.exp new file mode 100644 index 000000000..3934e5ab3 --- /dev/null +++ b/tests/functional2/lang/assets/test_lang_infra/runner_po.out.exp @@ -0,0 +1,2 @@ +_type: ExprSet +recursive: false diff --git a/tests/functional2/lang/lang_util.py b/tests/functional2/lang/lang_util.py new file mode 100644 index 000000000..31fb3cf48 --- /dev/null +++ b/tests/functional2/lang/lang_util.py @@ -0,0 +1,305 @@ +import logging +import re +from enum import StrEnum +from functools import cache +from pathlib import Path +from typing import Any + +import toml +from toml import TomlDecodeError + +from functional2.testlib.fixtures.file_helper import FileDeclaration, CopyFile, Symlink, RelativeTo +from functional2.testlib.utils import test_base_folder + + +LANG_TEST_ID_PATTERN = "{folder_name}:{test_name}" + + +class LangTestRunner(StrEnum): + """ + A list of possible runners for the lang tests on + The string value is the one present in the filenames / test.toml file as runner + """ + + EVAL_OKAY = "eval-okay" + EVAL_FAIL = "eval-fail" + PARSE_OKAY = "parse-okay" + PARSE_FAIL = "parse-fail" + + @classmethod + def as_regex_selector(cls) -> str: + """ + Returns a regex to match any available runner name into a named capturing group called "runner_name" + :return: a string containing a matching regex + """ + # For some reason ruff doesn't detect that .value is the string already and not a function + # it seems to work when accessing `LangTestRunner` but not when using `cls` + return rf"(?P{'|'.join([runner.value for runner in cls])})" # type: ignore + + +INVALID_TESTER_NAME = ( + f"invalid runner name: '%s', must be one of {[runner.value for runner in LangTestRunner]!r}" +) +""" +Base message for invalid runner, to use across collection +""" + +SUFFIX_REGEX = re.compile("-[\\w-]+?").pattern +NAMING_PATTERN_LANG_TEST = re.compile( + rf"{LangTestRunner.as_regex_selector()}(?P{SUFFIX_REGEX})?" +) + + +class LangTest: + def __init__( + self, + test_name: str, + folder_name: str, + runner: LangTestRunner, + flags: list[str] | None = None, + extra_files: list[str] | None = None, + suffix: str = "", + ): + """ + Internal class to represent a lang test + :param test_name: name of the explicit test (e.g. "eval-depr" or "eval-allow-depr") + :param folder_name: the folder / group of tests this one originates from (e.g. "nul_bytes") + :param runner: which runner to run this test on (e.g. EVAL_FAIL or PARSE_OKAY) + :param flags: additional flags provided for nix + :param extra_files: any additional files which should be copied into the tests directory + :param suffix: suffix of the in file + """ + self.test_name = test_name + self.full_name = LANG_TEST_ID_PATTERN.format( + folder_name=folder_name, test_name=f"{test_name}{suffix}" + ) + self.runner = runner + self.flags = flags or [] + self.folder = folder_name + self.extra_files = extra_files or [] + self.suffix = suffix + + def _get_files(self) -> FileDeclaration: + """ + Internal function to turn the metadata into a FileDeclaration used for parametrization + :return: FileDeclaration object containing all files required for the test + """ + files = { + "in.nix": CopyFile(f"{self.folder}/in{self.suffix}.nix"), + "lib.nix": CopyFile("lib.nix"), + "out.exp": Symlink( + f"{self.folder}/{self.test_name}{self.suffix}.out.exp", relative_to=RelativeTo.TEST + ), + "err.exp": Symlink( + f"{self.folder}/{self.test_name}{self.suffix}.err.exp", relative_to=RelativeTo.TEST + ), + } + for file in self.extra_files: + # Make sure to add the extra-files requested by the test.toml + files[file] = CopyFile(f"{self.folder}/{file}") + return files + + def to_params(self) -> tuple[FileDeclaration, list[str], str]: + """ + Converts the LangTest to the parameters required for parametrization of the test runners + :return: a Tuple of the FileDeclaration (used by the `files` fixture), list of flags and a unique id + """ + return self._get_files(), self.flags, self.full_name + + +class InvalidLangTest: + def __init__(self, name: str, reasons: list[str]): + """ + Metadata class for invalid test configuration, used to later fail a test with the given reasons + :param name: name of the test which is configured badly + :param reasons: a list of reasons as to why the configuration is invalid + """ + self.name = name + self.reasons = reasons + + +def _group_lang_tests(tests: list[LangTest]) -> dict[LangTestRunner, list[LangTest]]: + """ + groups the given list of tests by their runner + :param tests: list of tests to group + :return: grouped tests in the following order: EVAL_OKAY, EVAL_FAIL, PARSE_OKAY, PARSE_FAIL + """ + grouped_tests = {runner_name: [] for runner_name in LangTestRunner} + for test in tests: + grouped_tests[test.runner].append(test) + return grouped_tests + + +def _is_list_of_type(value: Any, expected_type: type[Any]) -> bool: + """ + checks if the given value conforms to the type `list[expected_type]` + :param value: value to check + :param expected_type: what type each item should be + :return: True, if it conforms, False otherwise + """ + return isinstance(value, list) and all(isinstance(v, expected_type) for v in value) + + +def _is_list_of_strings(value: Any) -> bool: + """ + same as `_is_list_of_type` but with type `str` pre-applied + :param value: value to check + :return: True, if the value conforms to `list[str]` otherwise False + """ + return _is_list_of_type(value, str) + + +def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]: + """ + Collects all tests, declared by a `test.toml` file within the given folder + :param folder: base folder to collect tests in + :return: a list of valid test configurations and a list of invalid test configurations + """ + parent_name = folder.name + test_declaration = folder / "test.toml" + try: + infos: dict[str, Any] = toml.load(test_declaration) + except TomlDecodeError as e: + return [], [InvalidLangTest(parent_name, [f"couldn't parse toml: {e!r}"])] + invalid_tests: list[InvalidLangTest] = [] + tests: list[LangTest] = [] + + # suffixes of the in files, e.g. + # in.nix => '' + # in-1.nix => '-1' + # in-some-test.nix => '-some-test' + # etc + in_suffixes = [ + suffix.group(1) or "" + for suffix in [ + re.fullmatch(rf"in({SUFFIX_REGEX})?\.nix", file.name) for file in folder.iterdir() + ] + if suffix is not None + ] + + for test_name, definition in infos.items(): + test_errors: list[str] = [] + full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name) + if not isinstance(definition, dict): + invalid_tests.append( + InvalidLangTest( + full_name, [f"invalid value for {test_name!r}; only tests are expected"] + ) + ) + continue + + flags = definition.pop("flags", []) + if not _is_list_of_strings(flags): + test_errors.append( + f"invalid value type for 'flags': {flags}, expected a list of strings" + ) + + runner_name = definition.pop("runner", None) + try: + runner = LangTestRunner(runner_name) + except ValueError: + test_errors.append(INVALID_TESTER_NAME % runner_name) + runner = None + + extra_files = definition.pop("extra-files", []) + if not _is_list_of_strings(extra_files): + test_errors.append( + f"invalid value type for 'extra_files': {extra_files}, expected a list of strings" + ) + + if len(definition) > 0: + test_errors.append(f"unexpected arguments: {list(definition.keys())!r}") + if test_errors: + invalid_tests.append(InvalidLangTest(full_name, test_errors)) + continue + + # Add a test for each in file + tests += [ + LangTest(test_name, folder.name, runner, flags, extra_files, in_suffix) + for in_suffix in in_suffixes + ] + + return tests, invalid_tests + + +def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]: + """ + Collects all tests within the given folder, if no `test.toml` file is presented for a more explicit test configuration + :param folder: base folder to collect tests in + :return: a list of valid test configurations and a list of invalid test configurations + """ + parent_name = folder.name + tests: list[LangTest] = [] + invalid_tests: list[InvalidLangTest] = [] + for file in folder.iterdir(): + file: Path + if file.suffix == ".exp": + # we cannot use `file.stem` here, as it only removes the last suffix. i.e. + # `"eval-okay.out.exp".stem` => "eval-okay.out" + # `"eval-okay.out.exp".split(".")[0]` => "eval-okay" + # or + # `"parse-fail-some-name.err.exp".stem` => "parse-fail-some-name.err" + # `"parse-fail-some-name.err.exp".split(".")[0]` => "parse-fail-some-name" + test_name = file.name.rsplit(".", 2)[0] + full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name) + match = re.fullmatch(NAMING_PATTERN_LANG_TEST, test_name) + if match is None: + if re.match(LangTestRunner.as_regex_selector(), test_name) is None: + reason = INVALID_TESTER_NAME % test_name + else: + reason = f"incorrectly formatted test name: {test_name!r}" + invalid_tests.append(InvalidLangTest(full_name, [reason])) + continue + runner_name, suffix = match.groups() + runner = LangTestRunner(runner_name) + tests.append(LangTest(runner_name, folder.name, runner, suffix=suffix or "")) + return tests, invalid_tests + + +def _collect_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]: + """ + Collects all tests within the given test group + :param folder: where the test group are located + :return: a list of valid test configurations and a list of invalid test configurations + """ + test_declaration = folder / "test.toml" + if test_declaration.exists(): + return _collect_toml_test_group(folder) + return _collect_generic_test_group(folder) + + +def _collect_all_tests() -> tuple[list[LangTest], list[InvalidLangTest]]: + """ + Collects all LangTests present in the functional2/lang folder + :return: a list of valid test configurations and a list of invalid test configurations + """ + logger = logging.getLogger("lang-test-collector") + lang_folder = test_base_folder / "functional2/lang" + tests: list[LangTest] = [] + invalid_tests: list[InvalidLangTest] = [] + for node in lang_folder.iterdir(): + node: Path + # skip files, as test groups are folders and custom tests will be collected by pytest + # these are files like this `lang_util.py` or the `lib.nix` etc + if node.is_file() or node.name == "assets": + continue + + # ignore test groups, which have a py file, as those are set up fully custom + # and expected to be collected by pytest and not by us + if len(list(node.glob("*.py"))): + logger.info("skipping %s as it contains a py file, assuming custom tests", node) + continue + t, i = _collect_test_group(node) + tests += t + invalid_tests += i + return tests, invalid_tests + + +@cache +def fetch_all_lang_tests() -> tuple[dict[LangTestRunner, list[LangTest]], list[InvalidLangTest]]: + """ + Collects all LangTests declared within `functional2/lang` and groups them by runner / type + :return: valid tests as a mapping of runner -> tests for said runner, InvalidLangTests + """ + tests, invalid_tests = _collect_all_tests() + return _group_lang_tests(tests), invalid_tests diff --git a/tests/functional2/lang/lib.nix b/tests/functional2/lang/lib.nix new file mode 120000 index 000000000..1284f096e --- /dev/null +++ b/tests/functional2/lang/lib.nix @@ -0,0 +1 @@ +../../functional/lang/lib.nix \ No newline at end of file diff --git a/tests/functional2/lang/test_lang.py b/tests/functional2/lang/test_lang.py new file mode 100644 index 000000000..d8c51a030 --- /dev/null +++ b/tests/functional2/lang/test_lang.py @@ -0,0 +1,124 @@ +import json +from collections.abc import Callable +from logging import Logger +from pathlib import Path + +import pytest +import yaml +from _pytest.fixtures import FixtureRequest +from _pytest.python import Metafunc + +from functional2.lang.lang_util import LangTest, fetch_all_lang_tests, LangTestRunner +from functional2.testlib.fixtures.nix import Nix +from functional2.testlib.fixtures.snapshot import Snapshot + + +def pytest_generate_tests(metafunc: Metafunc): + """ + This hook parametrizes the parser and eval tests with all test cases found in functional2/lang + :param metafunc: the test function to parametrize, provided by pytest + """ + func_name = metafunc.function.__name__ + tests, invalid_tests = fetch_all_lang_tests() + if func_name == "test_invalid_configuration": + metafunc.parametrize(("name", "reasons"), [(t.name, t.reasons) for t in invalid_tests]) + return + selected_runner: LangTestRunner + match func_name: + case "test_eval": + selected_runner = LangTestRunner.EVAL_OKAY + case "test_xfail_eval": + selected_runner = LangTestRunner.EVAL_FAIL + case "test_parser": + selected_runner = LangTestRunner.PARSE_OKAY + case "test_xfail_parser": + selected_runner = LangTestRunner.PARSE_FAIL + case _: + return + selected_tests = tests[selected_runner] + if len(selected_tests) > 0: + # ignoring type here, because map doesn't recognize the typing correctly + # due to returning multiple things and it expecting a single generic + files, flags, ids = zip(*map(LangTest.to_params, selected_tests)) # type: ignore + else: + files, flags, ids = [], [], [] + metafunc.parametrize(("files", "flags"), zip(files, flags), ids=ids, indirect=True) + + +@pytest.fixture +def flags(request: FixtureRequest) -> list[str]: + return request.param + + +def _cleanup_output(stdout: str, stderr: str, origin: Path) -> tuple[str, str]: + """ + Cleans up any paths present in stdout and stderr by replacing them with placeholders + """ + test_path = str(origin) + clean_out = stdout.replace(test_path, "/pwd") + clean_err = stderr.replace(test_path, "/pwd") + return clean_out, clean_err + + +def test_parser(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]): + nix_command = nix.nix_instantiate( + ["--parse", *flags, files / "in.nix"], + # TODO(Commentator2.0): Mirrors behavior of init.sh from functional + # keep this for migration, but make it declarative afterwards + # and only active for the tests which need it + flake=True, + ) + result = nix_command.run().ok() + stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files) + + # Taken from https://stackoverflow.com/a/39681672 + # Pyyaml does not give list items extra indentation, unlike pretty much everyone else. + # For migration compatibility with the `yq` output from the functional/lang tests, we + # use a custom dumper that produces the correct indentation. + # Upstream issue (open since 2018): https://github.com/yaml/pyyaml/issues/234 + class CustomFixedIndentationDumper(yaml.Dumper): + def increase_indent(self, flow: bool = False, indentless: bool = False): # noqa: ARG002 + super().increase_indent(flow, False) + + result_obj = json.loads(stdout) + result_yaml = yaml.dump( + result_obj, Dumper=CustomFixedIndentationDumper, default_flow_style=False + ) + # parser out are in a yaml format, which is why we convert it here for better viewing and editing + assert snapshot("out.exp") == result_yaml + assert snapshot("err.exp") == stderr + + +def test_xfail_parser(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]): + nix_command = nix.nix_instantiate(["--parse", *flags, files / "in.nix"], flake=True) + result = nix_command.run().expect(1) + stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files) + + assert snapshot("out.exp") == stdout + assert snapshot("err.exp") == stderr + + +def test_eval(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]): + nix_command = nix.nix_instantiate(["--eval", "--strict", *flags, files / "in.nix"], flake=True) + result = nix_command.run().ok() + stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files) + + assert snapshot("out.exp") == stdout + assert snapshot("err.exp") == stderr + + +def test_xfail_eval(files: Path, nix: Nix, flags: list[str], snapshot: Callable[[str], Snapshot]): + nix_command = nix.nix_instantiate( + ["--eval", "--strict", "--show-trace", *flags, files / "in.nix"], flake=True + ) + result = nix_command.run().expect(1) + stdout, stderr = _cleanup_output(result.stdout_s, result.stderr_s, files) + + assert snapshot("out.exp") == stdout + assert snapshot("err.exp") == stderr + + +def test_invalid_configuration(name: str, reasons: list[str], logger: Logger): + msg = f"Invalid configuration for {name!r}: {reasons}" + logger.error(msg) + pytest.fail(msg) diff --git a/tests/functional2/lang/test_lang_infra.py b/tests/functional2/lang/test_lang_infra.py new file mode 100644 index 000000000..02c1ad00e --- /dev/null +++ b/tests/functional2/lang/test_lang_infra.py @@ -0,0 +1,516 @@ +from collections.abc import Callable +from pathlib import Path +from textwrap import dedent + +import pytest +from functional2.testlib.commands import Command +from functional2.testlib.fixtures.file_helper import File, RelativeTo, Symlink +from functional2.testlib.fixtures.snapshot import Snapshot +from functional2.testlib.utils import get_functional2_lang_files + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "generic_test": { + "in.nix": File("{}"), + "eval-okay.out.exp": File("{ }\n"), + } + } + } + } + ), + ["-k", "generic_test", "--setup-plan"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_detects_generic_lang_test(pytest_command: Command): + result = pytest_command.run().ok() + assert "lang/test_lang.py::test_eval[generic_test:eval-okay]" in result.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "toml_test": { + "in.nix": File("{}"), + "my_name.out.exp": File("{ }\n"), + "test.toml": File( + dedent(""" + [my_name] + runner = "eval-okay" + """) + ), + } + } + } + } + ), + ["-k", "toml_test", "--setup-plan"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_detects_toml_lang_test(pytest_command: Command): + result = pytest_command.run().ok() + assert "lang/test_lang.py::test_eval[toml_test:my_name]" in result.stdout_plain + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "some_py_module": { + "in.nix": File("{}"), + "eval-okay.out.exp": File("{ }\n"), + "__init__.py": File(""), + } + } + } + } + ), + ["--setup-plan"], + ) + ], + indirect=True, +) +def test_skips_py_files(files: Path, pytest_command: Command): + result = pytest_command.run().ok() + assert "lang/test_lang.py::test_eval[some_py_module:eval-okay]" not in result.stdout_plain + assert ( + f"[ INFO] [lang-test-collector] skipping {files.absolute()}/functional2/lang/some_py_module as it contains a py file, assuming custom tests" + in result.stdout_plain + ) + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "n_suffix": { + "in-1.nix": File("{}"), + "in-2.nix": File("{}"), + "eval-okay-1.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + "eval-okay-2.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + } + } + } + } + ), + ["-k", "n_suffix"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_collects_with_n_suffix(pytest_command: Command): + result = pytest_command.run().ok() + out = result.stdout_plain + assert "n_suffix:eval-okay-1" in out + assert "n_suffix:eval-okay-2" in out + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "short_string_suffix": { + "in-speaker.nix": File("{}"), + "in-microphone.nix": File("{}"), + "eval-okay-speaker.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + "eval-okay-microphone.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + } + } + } + } + ), + ["-k", "short_string_suffix"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_collects_short_string_suffix(pytest_command: Command): + result = pytest_command.run().ok() + out = result.stdout_plain + assert "short_string_suffix:eval-okay-speaker" in out + assert "short_string_suffix:eval-okay-microphone" in out + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "dash_suffix": { + "in-string-with-dash.nix": File("{}"), + "in-some-more--dashes.nix": File("{}"), + "eval-okay-string-with-dash.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + "eval-okay-some-more--dashes.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + } + } + } + } + ), + ["-k", "dash_suffix"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_collects_string_suffix_with_dash(pytest_command: Command): + result = pytest_command.run().ok() + out = result.stdout_plain + assert "dash_suffix:eval-okay-string-with-dash" in out + assert "dash_suffix:eval-okay-some-more--dashes" in out + assert "dash_suffix:eval-okay-" in out + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "bad_naming": { + "in-&x.nix": File("{}"), + "in-.nix": File("{}"), + "eval-okay-&x.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + "eval-okay-.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + } + } + } + } + ), + ["-k", "bad_naming"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_collection_fails_with_bad_naming(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_plain + assert "bad_naming:eval-okay" in err + assert "incorrectly formatted test name: 'eval-okay-'" in err + assert "incorrectly formatted test name: 'eval-okay-&x'" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "infra_okay_runners": { + "in.nix": File("{}"), + "eval-okay.out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", + relative_to=RelativeTo.TEST, + ), + "parse-okay.out.exp": Symlink( + "assets/test_lang_infra/runner_po.out.exp", + relative_to=RelativeTo.TEST, + ), + }, + "infra_fail_runners": { + "in.nix": File("{"), + "eval-fail.err.exp": Symlink( + "assets/test_lang_infra/runner_ef.err.exp", + relative_to=RelativeTo.TEST, + ), + "parse-fail.err.exp": Symlink( + "assets/test_lang_infra/runner_pf.err.exp", + relative_to=RelativeTo.TEST, + ), + }, + } + } + } + ), + ["-k", "infra and runners"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_all_runners_work(pytest_command: Command): + result = pytest_command.run().ok() + assert "lang/test_lang.py::test_eval[infra_okay_runners:eval-okay]" in result.stdout_plain + assert "lang/test_lang.py::test_parser[infra_okay_runners:parse-okay]" in result.stdout_plain + assert "lang/test_lang.py::test_xfail_eval[infra_fail_runners:eval-fail]" in result.stdout_plain + assert ( + "lang/test_lang.py::test_xfail_parser[infra_fail_runners:parse-fail]" in result.stdout_plain + ) + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "generic_bad": { + "in.nix": File("{}"), + "fee-foo.out.exp": File(""), + "hello-okay.out.exp": File(""), + "parse-fops.out.exp": File(""), + } + } + } + } + ), + [], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_generic_bad_runner_name(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_s + assert "test_invalid_configuration" in err + assert "invalid runner name:" in err + assert "Invalid configuration for 'generic_bad:fee-foo'" in err + assert "Invalid configuration for 'generic_bad:hello-okay'" in err + assert "Invalid configuration for 'generic_bad:parse-fops'" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "toml_test": { + "in.nix": File("{}"), + "my_name.out.exp": File("{ }\n"), + "test.toml": File( + dedent(""" + [my_name] + runner = "plushies" + """) + ), + } + } + } + } + ), + ["-k", "toml_test"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_toml_bad_runner_name(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_plain + assert "test_invalid_configuration" in err + assert "invalid runner name:" in err + assert "Invalid configuration for 'toml_test:my_name':" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "toml_test": { + "in.nix": File("{}"), + "my_name.out.exp": File("{ }\n"), + "test.toml": File( + dedent(""" + [my_name] + runner = "eval-okay" + cuddles = true + """) + ), + } + } + } + } + ), + ["-k", "toml_test"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_toml_too_many_args(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_plain + assert "test_invalid_configuration" in err + assert "unexpected arguments: ['cuddles']" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "toml_test": { + "in.nix": File("{}"), + "my_name.out.exp": File("{ }\n"), + "test.toml": File( + dedent(""" + invalid_test = "eval-okay" + [my_name] + runner = "eval-okay" + flags = 1 + [second] + runner = "eval-okay" + extra-files = [false, true, true] + """) + ), + } + } + } + } + ), + ["-k", "toml_test"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_toml_invalid_argument_types(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_plain + assert "test_invalid_configuration" in err + assert "invalid value for 'invalid_test'; only tests are expected" in err + assert "invalid value type for 'flags'" in err + assert "invalid value type for 'extra_files':" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "toml_test": { + "in.nix": File("{}"), + "my_name.out.exp": File("{ }\n"), + "test.toml": File( + dedent(""" + [my_name] + runner = "eval-okay" + cuddles = True + """) + ), + } + } + } + } + ), + ["-k", "toml_test"], + ) + ], + indirect=True, +) +@pytest.mark.usefixtures("files") +def test_invalid_toml(pytest_command: Command): + result = pytest_command.run().expect(1) + err = result.stdout_plain + assert "test_invalid_configuration" in err + assert "couldn't parse toml" in err + + +@pytest.mark.parametrize( + ("files", "pytest_command"), + [ + ( + get_functional2_lang_files( + { + "functional2": { + "lang": { + "update_test": {"in.nix": File("{}"), "eval-okay.out.exp": File("old")} + } + }, + "out.exp": Symlink( + "assets/test_lang_infra/runner_eo.out.exp", relative_to=RelativeTo.TEST + ), + } + ), + (["-k", "update_test", "--accept-tests"], False), + ) + ], + indirect=True, +) +def test_updates_expected_output( + files: Path, pytest_command: Command, snapshot: Callable[[str], Snapshot] +): + assert (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() == "old" + pytest_command.run().ok() + assert ( + snapshot("out.exp") + == (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() + ) diff --git a/tests/functional2/meson.build b/tests/functional2/meson.build index b18d7035d..6365b2cb6 100644 --- a/tests/functional2/meson.build +++ b/tests/functional2/meson.build @@ -1,8 +1,12 @@ xdist_opts = [ # auto number of workers, max 12 jobs '-n', 'auto', '--maxprocesses=12', - # group tests by module or class; ensures that any setup work occurs as little as possible - '--dist=loadscope', + # Distributes tests evenly at first and then, if workers run out of tests, they steal from a different worker + # This is useful here, due to differing durations of tests, especially when tests check timeout functionality + # + # grouping by module/file is disadventageous here too, as all lang tests are executed from the same module + # which would result in high run times for that single worker while everything else is bored + '--dist=worksteal', ] # surprisingly, this actually works even if PATH is set to something before @@ -16,6 +20,8 @@ test( args : [ '-m', 'pytest', '-v', + # Print out summaries of **f**ailed, **E**rrored and **s**kipped tests + '-r', 'fEs', xdist_opts, meson.current_source_dir() ], diff --git a/tests/functional2/testlib/utils.py b/tests/functional2/testlib/utils.py index ee131b60e..576eb8bdb 100644 --- a/tests/functional2/testlib/utils.py +++ b/tests/functional2/testlib/utils.py @@ -66,3 +66,23 @@ def get_functional2_files_with_testlib( additional_files, ) ) + + +def get_functional2_lang_files(additional_files: FileDeclaration | None = None) -> FileDeclaration: + if additional_files is None: + additional_files = {} + return get_functional2_files_with_testlib( + merge_file_declaration( + { + "functional2": { + "lang": { + "__init__.py": CopyFile(functional2_base_folder / "lang/__init__.py"), + "lang_util.py": CopyFile(functional2_base_folder / "lang/lang_util.py"), + "test_lang.py": CopyFile(functional2_base_folder / "lang/test_lang.py"), + "lib.nix": CopyFile(functional2_base_folder / "lang/lib.nix"), + } + } + }, + additional_files, + ) + )