Commentator2.0
f7914e89e6
This creates a framework similar to the old lang.sh from functional. Some notable changes: - instead of having a .flags file, a test.toml can declare flags - additionally the test.toml can also declare extra files and multiple runners for the given input file. - there won't be any old tests hanging around anymore which weren't deleted properly in the installation - all files for a single test are defined decleratively and there won't be any residues Tests can be placed within the functional2/lang folder most migrations should be rather clean Implements: #825 Change-Id: I5f9149903ec5b078008969a4ae77305417c11475
517 lines
17 KiB
Python
517 lines
17 KiB
Python
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
from functional2.testlib.commands import Command
|
|
from functional2.testlib.fixtures.file_helper import File, RelativeTo, Symlink
|
|
from functional2.testlib.fixtures.snapshot import Snapshot
|
|
from functional2.testlib.utils import get_functional2_lang_files
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"generic_test": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": File("{ }\n"),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "generic_test", "--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_detects_generic_lang_test(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[generic_test:eval-okay]" in result.stdout_plain
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test", "--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_detects_toml_lang_test(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[toml_test:my_name]" in result.stdout_plain
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"some_py_module": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": File("{ }\n"),
|
|
"__init__.py": File(""),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_skips_py_files(files: Path, pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[some_py_module:eval-okay]" not in result.stdout_plain
|
|
assert (
|
|
f"[ INFO] [lang-test-collector] skipping {files.absolute()}/functional2/lang/some_py_module as it contains a py file, assuming custom tests"
|
|
in result.stdout_plain
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"n_suffix": {
|
|
"in-1.nix": File("{}"),
|
|
"in-2.nix": File("{}"),
|
|
"eval-okay-1.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"eval-okay-2.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "n_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_with_n_suffix(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "n_suffix:eval-okay-1" in out
|
|
assert "n_suffix:eval-okay-2" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"short_string_suffix": {
|
|
"in-speaker.nix": File("{}"),
|
|
"in-microphone.nix": File("{}"),
|
|
"eval-okay-speaker.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"eval-okay-microphone.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "short_string_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_short_string_suffix(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "short_string_suffix:eval-okay-speaker" in out
|
|
assert "short_string_suffix:eval-okay-microphone" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"dash_suffix": {
|
|
"in-string-with-dash.nix": File("{}"),
|
|
"in-some-more--dashes.nix": File("{}"),
|
|
"eval-okay-string-with-dash.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"eval-okay-some-more--dashes.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "dash_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_string_suffix_with_dash(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "dash_suffix:eval-okay-string-with-dash" in out
|
|
assert "dash_suffix:eval-okay-some-more--dashes" in out
|
|
assert "dash_suffix:eval-okay-" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"bad_naming": {
|
|
"in-&x.nix": File("{}"),
|
|
"in-.nix": File("{}"),
|
|
"eval-okay-&x.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"eval-okay-.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "bad_naming"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collection_fails_with_bad_naming(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "bad_naming:eval-okay" in err
|
|
assert "incorrectly formatted test name: 'eval-okay-'" in err
|
|
assert "incorrectly formatted test name: 'eval-okay-&x'" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"infra_okay_runners": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"parse-okay.out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_po.out.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
},
|
|
"infra_fail_runners": {
|
|
"in.nix": File("{"),
|
|
"eval-fail.err.exp": Symlink(
|
|
"assets/test_lang_infra/runner_ef.err.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
"parse-fail.err.exp": Symlink(
|
|
"assets/test_lang_infra/runner_pf.err.exp",
|
|
relative_to=RelativeTo.TEST,
|
|
),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "infra and runners"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_all_runners_work(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[infra_okay_runners:eval-okay]" in result.stdout_plain
|
|
assert "lang/test_lang.py::test_parser[infra_okay_runners:parse-okay]" in result.stdout_plain
|
|
assert "lang/test_lang.py::test_xfail_eval[infra_fail_runners:eval-fail]" in result.stdout_plain
|
|
assert (
|
|
"lang/test_lang.py::test_xfail_parser[infra_fail_runners:parse-fail]" in result.stdout_plain
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"generic_bad": {
|
|
"in.nix": File("{}"),
|
|
"fee-foo.out.exp": File(""),
|
|
"hello-okay.out.exp": File(""),
|
|
"parse-fops.out.exp": File(""),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
[],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_generic_bad_runner_name(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_s
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid runner name:" in err
|
|
assert "Invalid configuration for 'generic_bad:fee-foo'" in err
|
|
assert "Invalid configuration for 'generic_bad:hello-okay'" in err
|
|
assert "Invalid configuration for 'generic_bad:parse-fops'" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "plushies"
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_bad_runner_name(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid runner name:" in err
|
|
assert "Invalid configuration for 'toml_test:my_name':" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
cuddles = true
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_too_many_args(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "unexpected arguments: ['cuddles']" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
invalid_test = "eval-okay"
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
flags = 1
|
|
[second]
|
|
runner = "eval-okay"
|
|
extra-files = [false, true, true]
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_invalid_argument_types(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid value for 'invalid_test'; only tests are expected" in err
|
|
assert "invalid value type for 'flags'" in err
|
|
assert "invalid value type for 'extra_files':" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
cuddles = True
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_invalid_toml(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "couldn't parse toml" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"update_test": {"in.nix": File("{}"), "eval-okay.out.exp": File("old")}
|
|
}
|
|
},
|
|
"out.exp": Symlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp", relative_to=RelativeTo.TEST
|
|
),
|
|
}
|
|
),
|
|
(["-k", "update_test", "--accept-tests"], False),
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_updates_expected_output(
|
|
files: Path, pytest_command: Command, snapshot: Callable[[str], Snapshot]
|
|
):
|
|
assert (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() == "old"
|
|
pytest_command.run().ok()
|
|
assert (
|
|
snapshot("out.exp")
|
|
== (files / "functional2/lang/update_test/eval-okay.out.exp").read_text()
|
|
)
|