diff --git a/package.nix b/package.nix index 01778b2fd..de14bb139 100644 --- a/package.nix +++ b/package.nix @@ -248,6 +248,7 @@ let p.tappy p.ruff p.aiohttp + p.mistletoe (pyxattrForPython p) ]; diff --git a/tests/functional2/README.md b/tests/functional2/README.md index 59caa1474..05ab1fb25 100644 --- a/tests/functional2/README.md +++ b/tests/functional2/README.md @@ -17,6 +17,9 @@ When additional files are required for a test, they are placed within a folder ` The `lang` test package which contains all parser and evaluator tests is somewhat special, as it automatically discovers file-based tests from within subdirectories (see ["Writing lang tests"](#writing-lang-tests)). +The `repl_characterization` test package which contains interactive repl tests and is, similar to `lang`, somewhat special. +see ["Writing repl tests"](#writing-repl-tests) for further information. + Tests for the test suite itself are located in the `testlib` package. ## Running Tests @@ -382,3 +385,71 @@ Here too, it is possible to work with multiple input files, though it works slig - It is possible to call the according test runner function directly to avoid boilerplate - If additional functionalities are required, placing a `.py` file in the directory tells the framework to ignore it. One can then write [pytest tests](#writing-python-tests) as usual - The test suit will fail, if any files are unused. This is done to avoid unrecognized tests due to bad naming. + +## Writing Repl test +The `repl_characterization` tests work similar to the `lang` tests in the sense that tests aren't written directly in python, but in `.md` files instead and are auto-discovered by its framework. + +Similar to `lang`, each folder represents a test-group and can contain one or more files, which are separate tests. The files and folders can be named arbitrarily, as long as the files' extensions are `.md` + +Each `.md` file represents a single repl session. If you need to test things in multiple sessions, create multiple files. + +Expected outputs can be updated by running the tests with the `--accept-tests` flag enabled. + +### Input and output blocks +In order to pass input to a repl session, create a (fenced) codeblock using `nix` as its language. Paste in whatever commands or nix code you want to send to the repl. +Optionally, you may create another codeblock beneath, using `output` as its language to indicate its expected output. This block will otherwise be auto-generated upon running the test-suite with `--accept-tests` immediately below the input block. + +> Warning: +> Due to how the REPL is implemented, multiline code is **not** supported. The real command-line repl implemented it in a very sketcy way which is not replicatable with automation. + +For example: +`repl_characterization/my_test/example.md` +``````md +# This is an example for writing a repl test: + +Here we have some documenation. + +Below this we can see the input +```nix +1 + 1 +``` +The output will go below here: +```output +2 +``` + +# Another section, this is irrelevant for the test itself, just provides more doc +you can also use `~` for codeblocks BTW +~~~nix +f = a: a + ""; +~~~ +Mix and match all you want (input style = output style for auto-generation of output blocks) +````output +Added f. +```` +`````` + +### Repl options +If needed, one can change the following options using frontmatter: +- args: a list of strings added as cli arguments when the session is initialized. `{PWD}` will be replaced with the working directory. +- should_fail: boolean, if True indicates that the repl session should fail to initialize. When True, only a single `output` block is expected in the file. +- files: a list of relative paths for files to be accessible to the session + +These options are defined in the `ReplTestMetadata` class of `repl_util.py` + +For example: +````md +--- +args: ['--repl-overlays', '{PWD}/repl-overlay-fail.nix'] +should_fail: True +files: ['repl-overlay-fail.nix'] +--- + +```output +[error output omitted here] +``` + +## Additional notes: +Check `repl_basics/repl_basics.md` and `repl_overlay_errors/repl_overlay_errors.md` for more examples. +Everything which isn't frontmatter or a code-block with either `nix` or `output` as its language, will be considered a comment and ignored. +The current lix version will be replaced with `VERSION` auto-magically to ensure compatibility with newer versions/commits. diff --git a/tests/functional2/conftest.py b/tests/functional2/conftest.py index 678ec7c5b..6c0a3d9e4 100644 --- a/tests/functional2/conftest.py +++ b/tests/functional2/conftest.py @@ -10,6 +10,7 @@ pytest_plugins = ( "testlib.fixtures.nix", "testlib.fixtures.snapshot", "testlib.fixtures.pytest_command", + "testlib.repl_util", ) diff --git a/tests/functional2/repl_characterization/OWNERS b/tests/functional2/repl_characterization/OWNERS new file mode 100644 index 000000000..372c07b5c --- /dev/null +++ b/tests/functional2/repl_characterization/OWNERS @@ -0,0 +1,2 @@ +per-file *.md=* +per-file *.nix=* diff --git a/tests/functional2/repl_characterization/__init__.py b/tests/functional2/repl_characterization/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional2/repl_characterization/repl_basics/repl_basics.md b/tests/functional2/repl_characterization/repl_basics/repl_basics.md new file mode 100644 index 000000000..a35f1e44d --- /dev/null +++ b/tests/functional2/repl_characterization/repl_basics/repl_basics.md @@ -0,0 +1,90 @@ +# Basic repl test + +Disable error traces, because we have the `show-trace` setting enabled +```nix +:te +``` +```output +not showing error traces + +``` +trivial addition: + +```nix +1 + 1 +``` +```output +2 + +``` + +Lets check if docs work +```nix +:doc builtins.add +``` +```output +Synopsis: builtins.add e1 e2 + + + Return the sum of the numbers e1 and e2. + +``` + +## Error printing + +no trace by default +```nix +f = a: "" + a +f 2 +``` +```output +Added f. + +error: + … while concatenating + at «string»:1:11: + 1| f = a: "" + a + | ^ + + error: cannot coerce an integer to a string: 2 + +``` + +show trace when enabled +~~~nix +:te +f 2 +~~~ +~~~output +showing error traces + +error: + … from call site + at «string»:1:1: + 1| f 2 + | ^ + + … while calling anonymous lambda + at «string»:1:5: + 1| f = a: "" + a + | ^ + + … while concatenating + at «string»:1:11: + 1| f = a: "" + a + | ^ + + error: cannot coerce an integer to a string: 2 + +~~~ + +test if markdown does things correctly + +```nix +:p __replaceStrings ["a"] ["`"] "\naaa\n" +``` +````output + +``` + +```` diff --git a/tests/functional2/repl_characterization/repl_overlay_errors/repl-overlay-fail.nix b/tests/functional2/repl_characterization/repl_overlay_errors/repl-overlay-fail.nix new file mode 100644 index 000000000..426127916 --- /dev/null +++ b/tests/functional2/repl_characterization/repl_overlay_errors/repl-overlay-fail.nix @@ -0,0 +1 @@ +info: final: prev: builtins.abort "uh oh!" diff --git a/tests/functional2/repl_characterization/repl_overlay_errors/repl_overlay_errors.md b/tests/functional2/repl_characterization/repl_overlay_errors/repl_overlay_errors.md new file mode 100644 index 000000000..ac7acbd21 --- /dev/null +++ b/tests/functional2/repl_characterization/repl_overlay_errors/repl_overlay_errors.md @@ -0,0 +1,62 @@ +--- +args: ['--repl-overlays', '{PWD}/repl-overlay-fail.nix'] +should_fail: True +files: ['repl-overlay-fail.nix'] +--- + +`repl-overlays` that fail to evaluate should error. + +```output +Lix VERSION +Type :? for help. +Loading 'repl-overlays'... +error: + … while calling anonymous lambda + at «string»:2:16: + 1| + 2| info: initial: functions: + | ^ + 3| let + + … while evaluating final + at «string»:6:1: + 5| in + 6| final + | ^ + 7| + + … while calling the 'foldl'' builtin + at «string»:4:11: + 3| let + 4| final = builtins.foldl' (prev: function: prev // (function info final prev)) initial functions; + | ^ + 5| in + + … while calling anonymous lambda + at «string»:4:34: + 3| let + 4| final = builtins.foldl' (prev: function: prev // (function info final prev)) initial functions; + | ^ + 5| in + + … from call site + at «string»:4:53: + 3| let + 4| final = builtins.foldl' (prev: function: prev // (function info final prev)) initial functions; + | ^ + 5| in + + … while calling anonymous lambda + at /pwd/repl-overlay-fail.nix:1:14: + 1| info: final: prev: builtins.abort "uh oh!" + | ^ + 2| + + … while calling the 'abort' builtin + at /pwd/repl-overlay-fail.nix:1:20: + 1| info: final: prev: builtins.abort "uh oh!" + | ^ + 2| + + error: evaluation aborted with the following error message: 'uh oh!' +``` diff --git a/tests/functional2/repl_characterization/test_repl.py b/tests/functional2/repl_characterization/test_repl.py new file mode 100644 index 000000000..ff5e5b4d3 --- /dev/null +++ b/tests/functional2/repl_characterization/test_repl.py @@ -0,0 +1,38 @@ +import re +from pathlib import Path + +import pytest +from mistletoe.markdown_renderer import MarkdownRenderer + +from testlib.repl_util import ReplTest, ReplTestMetadata, get_repl_test_params +from testlib.fixtures.nix import Nix + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + if metafunc.definition.name != "test_repl_char": + return + + params, ids = get_repl_test_params() + metafunc.parametrize(("files", "metadata"), params, indirect=["files"], ids=ids) + + +def _clean_output(output: str, origin: Path) -> str: + lix_version_regex = r"Lix \d+\.\d+\.\d+-?[^\n ]*" + return re.sub(lix_version_regex, "Lix VERSION", output.replace(str(origin), "/pwd")) + + +def test_repl_char(nix: Nix, do_snapshot_update: bool, metadata: ReplTestMetadata, files: Path): + nix.settings.add_xp_feature("nix-command", "flakes", "repl-automation") + with MarkdownRenderer() as renderer: + test: ReplTest = metadata.create_test() + + args = [arg.replace("{PWD}", str(files.absolute())) for arg in metadata.args or []] + cmd = nix.nix(["repl", "--offline", *args]).with_stdin(test.input.encode()) + cmd.err_to_out = True + res = cmd.run().expect(metadata.should_fail or 0) + + usable_output = _clean_output(res.stdout_plain, files) + if test.check_and_update(usable_output, do_snapshot_update): + new_content = metadata.as_frontmatter + renderer.render(test.doc) + metadata.file.write_text(new_content) + pytest.skip("Updated golden files") diff --git a/tests/functional2/repl_characterization/test_repl_infra.py b/tests/functional2/repl_characterization/test_repl_infra.py new file mode 100644 index 000000000..f6c46f9e0 --- /dev/null +++ b/tests/functional2/repl_characterization/test_repl_infra.py @@ -0,0 +1,138 @@ +from textwrap import dedent +from pathlib import Path +from testlib.utils import functional2_base_folder +from testlib.fixtures.file_helper import CopyFile +from testlib.fixtures.file_helper import merge_file_declaration +from testlib.utils import get_functional2_files_with_testlib +from testlib.fixtures.file_helper import FileDeclaration +from testlib.fixtures.file_helper import File +from testlib.fixtures.file_helper import with_files +import pytest +from testlib.fixtures.command import Command + + +def get_functional2_repl_files(files: FileDeclaration | None = None) -> FileDeclaration: + repl_base = functional2_base_folder / "repl_characterization" + files = {"functional2": {"repl_characterization": files or {}}} + total_files = merge_file_declaration( + files, + { + "functional2": { + "repl_characterization": { + "__init__.py": File(""), + "test_repl.py": CopyFile(repl_base / "test_repl.py"), + } + } + }, + ) + return get_functional2_files_with_testlib(total_files) + + +@pytest.mark.parametrize("pytest_command", [["-k", "repl_char"]], indirect=True) +@with_files( + get_functional2_repl_files( + { + "repl_basics": { + "nya.md": File( + dedent(""" + ```nix + 1+1 + ``` + ```output + 2 + + ``` + """) + ) + } + } + ) +) +def test_trivial_succeeds(pytest_command: Command): + res = pytest_command.run().ok() + assert "repl_basics:nya.md] PASSED" in res.stdout_s + + +@pytest.mark.parametrize("pytest_command", [(["-k", "repl_char"], False)], indirect=True) +@with_files( + get_functional2_repl_files( + { + "repl_basics": { + "nya.md": File( + dedent(""" + ```nix + 1+1 + ``` + ```output + 3 + + ``` + """) + ) + } + } + ) +) +def test_trivial_fails(pytest_command: Command): + res = pytest_command.run().expect(1) + assert ( + "FAILED repl_characterization/test_repl.py::test_repl_char[repl_basics:nya.md]" + in res.stdout_s + ) + + +@pytest.mark.parametrize("pytest_command", [["-k", "repl_char", "--accept-tests"]], indirect=True) +@with_files( + get_functional2_repl_files( + { + "repl_basics": { + "nya.md": File( + dedent(""" + ```nix + 1+1 + ``` + ```output + 3 + + ``` + """) + ) + } + } + ) +) +def test_updates(pytest_command: Command, files: Path): + md_file = files / "functional2" / "repl_characterization" / "repl_basics" / "nya.md" + assert "output\n3" in md_file.read_text() + pytest_command.run().ok() + assert "output\n3" not in md_file.read_text() + assert "output\n2" in md_file.read_text() + + +@pytest.mark.parametrize("pytest_command", [["-k", "repl_char", "--accept-tests"]], indirect=True) +@with_files( + get_functional2_repl_files( + { + "repl_basics": { + "nya.md": File( + dedent(""" + ```nix + 1+1 + ``` + """) + ) + } + } + ) +) +def test_trivial_creates_block(pytest_command: Command, files: Path): + f = files / "functional2" + assert f.exists() + f /= "repl_characterization" + assert f.exists() + f /= "repl_basics" + assert f.exists() + md_file = files / "functional2" / "repl_characterization" / "repl_basics" / "nya.md" + assert "output" not in md_file.read_text() + pytest_command.run().ok() + assert "```output\n2\n\n```" in md_file.read_text() diff --git a/tests/functional2/testlib/repl_util.py b/tests/functional2/testlib/repl_util.py new file mode 100644 index 000000000..6aab703ad --- /dev/null +++ b/tests/functional2/testlib/repl_util.py @@ -0,0 +1,213 @@ +from mistletoe.markdown_renderer import BlankLine +from typing import Any +from _pytest.config import Config +import mistletoe +import dataclasses +import re +from dataclasses import dataclass +from pathlib import Path + +import frontmatter +from mistletoe.block_token import BlockToken, CodeFence + +from testlib.fixtures.file_helper import CopyFile, FileDeclaration +from testlib.utils import functional2_base_folder + + +def _add_output_codefence(input_elem: CodeFence) -> CodeFence: + parent = input_elem.parent + pos = parent.children.index(input_elem) + match = ("", (input_elem.indentation, input_elem.delimiter, "output", input_elem.language)) + output_block = CodeFence(match) + parent.children.insert(pos + 1, output_block) + parent.children.insert(pos + 2, BlankLine("\n")) + return output_block + + +@dataclasses.dataclass +class ReplTestMetadata: + args: list[str] | None + should_fail: bool | None + files: list[str] | None + + content: str + unknown: dict[str, str] + file: Path + + @classmethod + def keys_in_file(cls) -> set[str]: + return ["args", "should_fail", "files"] + + @property + def as_frontmatter(self) -> str: + meta = dataclasses.asdict(self) + file_items = {k: meta[k] for k in self.keys_in_file()} + data = [f"{k}: {v}" for k, v in file_items.items() if v is not None] + return "\n".join(["---", *data, "---", "\n"]) if data else "" + + def create_test(self) -> "ReplTest": + exceptions = [] + + if self.unknown: + exceptions.append( + ValueError( + f"Found unknown metadata: {self.unknown}\nValid metadata attributes are: {ReplTestMetadata.keys_in_file}" + ) + ) + + blocks = [] + doc = mistletoe.Document(self.content) + current_input: CodeFence | None = None + for elem in list(doc.children): + elem: BlockToken + if not isinstance(elem, CodeFence): + continue + + match elem.language: + case "nix": + if current_input: + blocks.append( + ReplTestBlock( + current_input.content, _add_output_codefence(current_input) + ) + ) + current_input = elem + case "output": + if current_input: + blocks.append(ReplTestBlock(current_input.content, elem)) + current_input = None + elif self.should_fail: + blocks.append(ReplTestBlock("", elem)) + else: + exceptions.append( + ValueError( + f"Found output block without input block at line {elem.line_number}" + ) + ) + + if current_input: + blocks.append( + ReplTestBlock(current_input.content, _add_output_codefence(current_input)) + ) + + if not blocks: + exceptions.append(ValueError("not test input (or output) found")) + + if exceptions: + raise ExceptionGroup("Invalid Test configuration:", exceptions) + return ReplTest(blocks, self, doc) + + +@dataclass +class ReplTestBlock: + input: str + output: CodeFence + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return False + return self.output.content.strip() == other.strip() + + +@dataclass +class ReplTest: + blocks: list[ReplTestBlock] + metadata: ReplTestMetadata + doc: mistletoe.Document + + @property + def input(self) -> str: + return "\n".join(block.input.strip() for block in self.blocks) + + def check_and_update(self, output: str, do_update: bool) -> bool: + updated = False + for actual, expected in self._output_to_blocks(output): + if not do_update: + assert expected == actual + else: + if expected == actual: + continue + updated = True + expected = expected.output + # HACK(rootile, 2026-05): The \n is required due to the renderer seeming to have an off-by-one error resulting in the deletion of the last character :melt: + appendix = "" if actual.endswith("\n") else "\n" + expected.children[0].content = actual + appendix + delimiter_length = ( + max(len(line) for line in actual.splitlines() if all(c == "`" for c in line)) + + 1 + ) + expected.delimiter = "`" * max(delimiter_length, 3) + return updated + + def _output_to_blocks(self, output: str) -> list[tuple[str, ReplTestBlock]]: + if self.metadata.should_fail: + return [(output, self.blocks[0])] + blocks = [] + # Remove the First output, as this will always be the lix version + output = output.split("\x05")[1:] + for block in self.blocks: + tasks = block.input.count("\n") + test_output, output = output[:tasks], output[tasks:] + test_output = "".join(test_output) + test_output = re.sub(r"^\s+$", "\n", test_output, flags=re.MULTILINE) + blocks.append(("".join(test_output), block)) + + return blocks + + +def _collect_repl_tests() -> list[Path]: + return [ + file + for test_folder in (functional2_base_folder / "repl_characterization").iterdir() + if test_folder.is_dir() + for file in test_folder.iterdir() + if file.suffix == ".md" + ] + + +def get_repl_test_params() -> tuple[list[tuple[FileDeclaration, ReplTestMetadata]]]: + params = [] + ids = [] + for file in _collect_repl_tests(): + fm = frontmatter.loads(file.read_text()) + metadata = fm.metadata + should_fail = metadata.pop("should_fail", None) + args = metadata.pop("args", None) + files = metadata.pop("files", None) + files_param = {f: CopyFile(file.parent / f) for f in files or {}} + params.append( + (files_param, ReplTestMetadata(args, should_fail, files, fm.content, metadata, file)) + ) + ids.append(f"{file.parent.name}:{file.name}") + return params, ids + + +def pytest_assertrepr_compare(config: Config, op: str, left: Any, right: Any) -> list[str] | None: + if not isinstance(left, ReplTestBlock) or op != "==" or not isinstance(right, str): + return None + left: ReplTestBlock + right: str + + exp_lines = left.output.content.strip().splitlines() + act_lines = right.strip().splitlines() + expl = [ + f"repl output of {left.input!r} differs. Consider using `--accept-tests` to update the golden files." + ] + if config.get_verbosity(): + expl.append("The following lines were mismatched:") + l_exp = len(exp_lines) + l_act = len(act_lines) + for i in range(max(l_exp, l_act)): + exp = exp_lines[i] if i < l_exp else None + act = act_lines[i] if i < l_act else None + if i > l_exp - 1: + expl.append(f"{i + 1}: + {act}") + continue + if i > l_act - 1: + expl.append(f"{i + 1}: - {exp}") + continue + if exp != act: + expl.append(f"{i + 1}: - {exp}") + expl.append(f"{i + 1}: + {act}") + + return expl