piegames
a7e5ff0070
The current `RelativeTo` design is both more complex and more confusing than necessary. Its four variants are now reduced to only two. They are now also represented as different classes, to better communicate the difference in semantics and also intent. Change-Id: Ia60fc7a2dfa0f62bdef90dde347fd8603fd3fbf9
503 lines
17 KiB
Python
503 lines
17 KiB
Python
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
from functional2.testlib.commands import Command
|
|
from functional2.testlib.fixtures.file_helper import File, AssetSymlink
|
|
from functional2.testlib.fixtures.snapshot import Snapshot
|
|
from functional2.testlib.utils import get_functional2_lang_files
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"generic_test": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": File("{ }\n"),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "generic_test", "--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_detects_generic_lang_test(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[generic_test:eval-okay]" in result.stdout_plain
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test", "--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_detects_toml_lang_test(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[toml_test:my_name]" in result.stdout_plain
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"some_py_module": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": File("{ }\n"),
|
|
"__init__.py": File(""),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["--setup-plan"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_skips_py_files(files: Path, pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[some_py_module:eval-okay]" not in result.stdout_plain
|
|
assert (
|
|
f"[ INFO] [lang-test-collector] skipping {files.absolute()}/functional2/lang/some_py_module as it contains a py file, assuming custom tests"
|
|
in result.stdout_plain
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"n_suffix": {
|
|
"in-1.nix": File("{}"),
|
|
"in-2.nix": File("{}"),
|
|
"eval-okay-1.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
"eval-okay-2.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "n_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_with_n_suffix(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "n_suffix:eval-okay-1" in out
|
|
assert "n_suffix:eval-okay-2" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"short_string_suffix": {
|
|
"in-speaker.nix": File("{}"),
|
|
"in-microphone.nix": File("{}"),
|
|
"eval-okay-speaker.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
"eval-okay-microphone.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "short_string_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_short_string_suffix(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "short_string_suffix:eval-okay-speaker" in out
|
|
assert "short_string_suffix:eval-okay-microphone" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"dash_suffix": {
|
|
"in-string-with-dash.nix": File("{}"),
|
|
"in-some-more--dashes.nix": File("{}"),
|
|
"eval-okay-string-with-dash.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
"eval-okay-some-more--dashes.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "dash_suffix"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collects_string_suffix_with_dash(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
out = result.stdout_plain
|
|
assert "dash_suffix:eval-okay-string-with-dash" in out
|
|
assert "dash_suffix:eval-okay-some-more--dashes" in out
|
|
assert "dash_suffix:eval-okay-" in out
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"bad_naming": {
|
|
"in-&x.nix": File("{}"),
|
|
"in-.nix": File("{}"),
|
|
"eval-okay-&x.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
"eval-okay-.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "bad_naming"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_collection_fails_with_bad_naming(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "bad_naming:eval-okay" in err
|
|
assert "incorrectly formatted test name: 'eval-okay-'" in err
|
|
assert "incorrectly formatted test name: 'eval-okay-&x'" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"infra_okay_runners": {
|
|
"in.nix": File("{}"),
|
|
"eval-okay.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_eo.out.exp"
|
|
),
|
|
"parse-okay.out.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_po.out.exp"
|
|
),
|
|
},
|
|
"infra_fail_runners": {
|
|
"in.nix": File("{"),
|
|
"eval-fail.err.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_ef.err.exp"
|
|
),
|
|
"parse-fail.err.exp": AssetSymlink(
|
|
"assets/test_lang_infra/runner_pf.err.exp"
|
|
),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "infra and runners"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_all_runners_work(pytest_command: Command):
|
|
result = pytest_command.run().ok()
|
|
assert "lang/test_lang.py::test_eval[infra_okay_runners:eval-okay]" in result.stdout_plain
|
|
assert "lang/test_lang.py::test_parser[infra_okay_runners:parse-okay]" in result.stdout_plain
|
|
assert "lang/test_lang.py::test_xfail_eval[infra_fail_runners:eval-fail]" in result.stdout_plain
|
|
assert (
|
|
"lang/test_lang.py::test_xfail_parser[infra_fail_runners:parse-fail]" in result.stdout_plain
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"generic_bad": {
|
|
"in.nix": File("{}"),
|
|
"fee-foo.out.exp": File(""),
|
|
"hello-okay.out.exp": File(""),
|
|
"parse-fops.out.exp": File(""),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
[],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_generic_bad_runner_name(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_s
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid runner name:" in err
|
|
assert "Invalid configuration for 'generic_bad:fee-foo'" in err
|
|
assert "Invalid configuration for 'generic_bad:hello-okay'" in err
|
|
assert "Invalid configuration for 'generic_bad:parse-fops'" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "plushies"
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_bad_runner_name(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid runner name:" in err
|
|
assert "Invalid configuration for 'toml_test:my_name':" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
cuddles = true
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_too_many_args(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "unexpected arguments: ['cuddles']" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
invalid_test = "eval-okay"
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
flags = 1
|
|
[second]
|
|
runner = "eval-okay"
|
|
extra-files = [false, true, true]
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_toml_invalid_argument_types(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "invalid value for 'invalid_test'; only tests are expected" in err
|
|
assert "invalid value type for 'flags'" in err
|
|
assert "invalid value type for 'extra_files':" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"toml_test": {
|
|
"in.nix": File("{}"),
|
|
"my_name.out.exp": File("{ }\n"),
|
|
"test.toml": File(
|
|
dedent("""
|
|
[my_name]
|
|
runner = "eval-okay"
|
|
cuddles = True
|
|
""")
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
),
|
|
["-k", "toml_test"],
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
@pytest.mark.usefixtures("files")
|
|
def test_invalid_toml(pytest_command: Command):
|
|
result = pytest_command.run().expect(1)
|
|
err = result.stdout_plain
|
|
assert "test_invalid_configuration" in err
|
|
assert "couldn't parse toml" in err
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("files", "pytest_command"),
|
|
[
|
|
(
|
|
get_functional2_lang_files(
|
|
{
|
|
"functional2": {
|
|
"lang": {
|
|
"update_test": {"in.nix": File("{}"), "eval-okay.out.exp": File("old")}
|
|
}
|
|
},
|
|
"out.exp": AssetSymlink("assets/test_lang_infra/runner_eo.out.exp"),
|
|
}
|
|
),
|
|
(["-k", "update_test", "--accept-tests"], False),
|
|
)
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_updates_expected_output(
|
|
files: Path, pytest_command: Command, snapshot: Callable[[str], Snapshot]
|
|
):
|
|
assert (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() == "old"
|
|
pytest_command.run().ok()
|
|
assert (
|
|
snapshot("out.exp")
|
|
== (files / "functional2/lang/update_test/eval-okay.out.exp").read_text()
|
|
)
|