from collections.abc import Callable from pathlib import Path from textwrap import dedent import pytest from functional2.testlib.commands import Command from functional2.testlib.fixtures.file_helper import File, AssetSymlink from functional2.testlib.fixtures.snapshot import Snapshot from functional2.testlib.utils import get_functional2_lang_files @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "generic_test": { "in.nix": File("{}"), "eval-okay.out.exp": File("{ }\n"), } } } } ), ["-k", "generic_test", "--setup-plan"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_detects_generic_lang_test(pytest_command: Command): result = pytest_command.run().ok() assert "lang/test_lang.py::test_eval[generic_test:eval-okay]" in result.stdout_plain @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "toml_test": { "in.nix": File("{}"), "my_name.out.exp": File("{ }\n"), "test.toml": File( dedent(""" [my_name] runner = "eval-okay" """) ), } } } } ), ["-k", "toml_test", "--setup-plan"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_detects_toml_lang_test(pytest_command: Command): result = pytest_command.run().ok() assert "lang/test_lang.py::test_eval[toml_test:my_name]" in result.stdout_plain @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "some_py_module": { "in.nix": File("{}"), "eval-okay.out.exp": File("{ }\n"), "__init__.py": File(""), } } } } ), ["--setup-plan"], ) ], indirect=True, ) def test_skips_py_files(files: Path, pytest_command: Command): result = pytest_command.run().ok() assert "lang/test_lang.py::test_eval[some_py_module:eval-okay]" not in result.stdout_plain assert ( f"[ INFO] [lang-test-collector] skipping {files.absolute()}/functional2/lang/some_py_module as it contains a py file, assuming custom tests" in result.stdout_plain ) @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "n_suffix": { "in-1.nix": File("{}"), "in-2.nix": File("{}"), "eval-okay-1.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), "eval-okay-2.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), } } } } ), ["-k", "n_suffix"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_collects_with_n_suffix(pytest_command: Command): result = pytest_command.run().ok() out = result.stdout_plain assert "n_suffix:eval-okay-1" in out assert "n_suffix:eval-okay-2" in out @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "short_string_suffix": { "in-speaker.nix": File("{}"), "in-microphone.nix": File("{}"), "eval-okay-speaker.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), "eval-okay-microphone.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), } } } } ), ["-k", "short_string_suffix"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_collects_short_string_suffix(pytest_command: Command): result = pytest_command.run().ok() out = result.stdout_plain assert "short_string_suffix:eval-okay-speaker" in out assert "short_string_suffix:eval-okay-microphone" in out @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "dash_suffix": { "in-string-with-dash.nix": File("{}"), "in-some-more--dashes.nix": File("{}"), "eval-okay-string-with-dash.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), "eval-okay-some-more--dashes.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), } } } } ), ["-k", "dash_suffix"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_collects_string_suffix_with_dash(pytest_command: Command): result = pytest_command.run().ok() out = result.stdout_plain assert "dash_suffix:eval-okay-string-with-dash" in out assert "dash_suffix:eval-okay-some-more--dashes" in out assert "dash_suffix:eval-okay-" in out @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "bad_naming": { "in-&x.nix": File("{}"), "in-.nix": File("{}"), "eval-okay-&x.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), "eval-okay-.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), } } } } ), ["-k", "bad_naming"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_collection_fails_with_bad_naming(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_plain assert "bad_naming:eval-okay" in err assert "incorrectly formatted test name: 'eval-okay-'" in err assert "incorrectly formatted test name: 'eval-okay-&x'" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "infra_okay_runners": { "in.nix": File("{}"), "eval-okay.out.exp": AssetSymlink( "assets/test_lang_infra/runner_eo.out.exp" ), "parse-okay.out.exp": AssetSymlink( "assets/test_lang_infra/runner_po.out.exp" ), }, "infra_fail_runners": { "in.nix": File("{"), "eval-fail.err.exp": AssetSymlink( "assets/test_lang_infra/runner_ef.err.exp" ), "parse-fail.err.exp": AssetSymlink( "assets/test_lang_infra/runner_pf.err.exp" ), }, } } } ), ["-k", "infra and runners"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_all_runners_work(pytest_command: Command): result = pytest_command.run().ok() assert "lang/test_lang.py::test_eval[infra_okay_runners:eval-okay]" in result.stdout_plain assert "lang/test_lang.py::test_parser[infra_okay_runners:parse-okay]" in result.stdout_plain assert "lang/test_lang.py::test_xfail_eval[infra_fail_runners:eval-fail]" in result.stdout_plain assert ( "lang/test_lang.py::test_xfail_parser[infra_fail_runners:parse-fail]" in result.stdout_plain ) @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "generic_bad": { "in.nix": File("{}"), "fee-foo.out.exp": File(""), "hello-okay.out.exp": File(""), "parse-fops.out.exp": File(""), } } } } ), [], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_generic_bad_runner_name(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_s assert "test_invalid_configuration" in err assert "invalid runner name:" in err assert "Invalid configuration for 'generic_bad:fee-foo'" in err assert "Invalid configuration for 'generic_bad:hello-okay'" in err assert "Invalid configuration for 'generic_bad:parse-fops'" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "toml_test": { "in.nix": File("{}"), "my_name.out.exp": File("{ }\n"), "test.toml": File( dedent(""" [my_name] runner = "plushies" """) ), } } } } ), ["-k", "toml_test"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_toml_bad_runner_name(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_plain assert "test_invalid_configuration" in err assert "invalid runner name:" in err assert "Invalid configuration for 'toml_test:my_name':" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "toml_test": { "in.nix": File("{}"), "my_name.out.exp": File("{ }\n"), "test.toml": File( dedent(""" [my_name] runner = "eval-okay" cuddles = true """) ), } } } } ), ["-k", "toml_test"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_toml_too_many_args(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_plain assert "test_invalid_configuration" in err assert "unexpected arguments: ['cuddles']" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "toml_test": { "in.nix": File("{}"), "my_name.out.exp": File("{ }\n"), "test.toml": File( dedent(""" invalid_test = "eval-okay" [my_name] runner = "eval-okay" flags = 1 [second] runner = "eval-okay" extra-files = [false, true, true] """) ), } } } } ), ["-k", "toml_test"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_toml_invalid_argument_types(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_plain assert "test_invalid_configuration" in err assert "invalid value for 'invalid_test'; only tests are expected" in err assert "invalid value type for 'flags'" in err assert "invalid value type for 'extra_files':" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "toml_test": { "in.nix": File("{}"), "my_name.out.exp": File("{ }\n"), "test.toml": File( dedent(""" [my_name] runner = "eval-okay" cuddles = True """) ), } } } } ), ["-k", "toml_test"], ) ], indirect=True, ) @pytest.mark.usefixtures("files") def test_invalid_toml(pytest_command: Command): result = pytest_command.run().expect(1) err = result.stdout_plain assert "test_invalid_configuration" in err assert "couldn't parse toml" in err @pytest.mark.parametrize( ("files", "pytest_command"), [ ( get_functional2_lang_files( { "functional2": { "lang": { "update_test": {"in.nix": File("{}"), "eval-okay.out.exp": File("old")} } }, "out.exp": AssetSymlink("assets/test_lang_infra/runner_eo.out.exp"), } ), (["-k", "update_test", "--accept-tests"], False), ) ], indirect=True, ) def test_updates_expected_output( files: Path, pytest_command: Command, snapshot: Callable[[str], Snapshot] ): assert (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() == "old" pytest_command.run().ok() assert ( snapshot("out.exp") == (files / "functional2/lang/update_test/eval-okay.out.exp").read_text() )