tests/functional2/lang: require all files to be used

Added an additional check that all files present within a folder must be
used/referenced. Otherwise an InvalidLangTest will be created.
This ensures that there weren't any mishaps while migrating tests
resulting in files being ignored and hence some tests not being run.

Fixes: #852

Change-Id: Ie096c5670bc20325ba72c7d6ce33c06667c66ab1

tests/functional2/README.md

@@ -355,3 +355,4 @@ Here too, it is possible to work with multiple input files, though it works slig
 - In the `test.toml`, it is currently not supported to pass paths with subdirectories into the `extra-files` attribute. If that functionality is required, use a [pytest tests](#writing-python-tests) instead.
   - It is possible to call the according test runner function directly to avoid boilerplate
 - If additional functionalities are required, placing a `.py` file in the directory tells the framework to ignore it. One can then write [pytest tests](#writing-python-tests) as usual
+- The test suit will fail, if any files are unused. This is done to avoid unrecognized tests due to bad naming.

tests/functional2/lang/lang_util.py

@@ -299,6 +299,8 @@ def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[Invalid
     # files starting with "_" will be ignored, e.g. "__pycache__"
     all_files = {f.name for f in folder.iterdir() if not f.name.startswith("_")}
     in_files = [f for f in all_files if re.fullmatch(rf"in({SUFFIX_REGEX})?\.nix", f) is not None]
+    # used to make sure all files are actually used
+    unused_files = all_files - {"test.toml"}
 
     try:
         infos: dict[str, Any] = toml.load(folder / "test.toml")
@@ -317,6 +319,16 @@ def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[Invalid
         tests += new_tests
         invalid_tests += new_invalids
 
+        unused_files -= set(test.extra_files)
+        for t in new_tests:
+            unused_files -= {t.in_file_name, f"{t.test_name}.out.exp", f"{t.test_name}.err.exp"}
+
+    if len(unused_files) > 0:
+        invalid_tests.append(
+            InvalidLangTest(
+                parent_name, [f"the following files weren't referenced: {unused_files!r}"]
+            )
+        )
     return tests, invalid_tests
 
 
@@ -329,7 +341,9 @@ def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[Inva
     parent_name = folder.name
     tests: list[LangTest] = []
     invalid_tests: list[InvalidLangTest] = []
-    for file in folder.iterdir():
+    all_files = set(folder.iterdir())
+    unused = {f.name for f in all_files if not f.name.startswith("_")}
+    for file in all_files:
         file: Path
         if file.suffix == ".exp":
             # we cannot use `file.stem` here, as it only removes the last suffix. i.e.
@@ -355,6 +369,11 @@ def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[Inva
             tests.append(
                 LangTest(runner_name, folder.name, runner, InFile(f"in{suffix}.nix", suffix))
             )
+            unused -= {file.name, f"in{suffix or ''}.nix"}
+    if len(unused) > 0:
+        invalid_tests.append(
+            InvalidLangTest(parent_name, [f"the following files weren't referenced: {unused!r}"])
+        )
     return tests, invalid_tests
 
 

tests/functional2/lang/test_lang_infra.py

@@ -508,6 +508,77 @@ def test_updates_expected_output(
     )
 
 
+@pytest.mark.parametrize(
+    ("files", "pytest_command"),
+    [
+        (
+            get_functional2_lang_files(
+                {
+                    "functional2": {
+                        "lang": {
+                            "generic_unused": {
+                                "in.nix": File("{}"),
+                                "in-1.nix": File("{}"),
+                                "eval-okay.out.exp": AssetSymlink(
+                                    "assets/test_lang_infra/runner_eo.out.exp"
+                                ),
+                            }
+                        }
+                    }
+                }
+            ),
+            [],
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.usefixtures("files")
+def test_generic_throws_unused_files(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    out = res.stdout_plain
+    assert "test_invalid_configuration[generic_unused-reasons0]" in out
+    assert "the following files weren't referenced: {'in-1.nix'}" in out
+
+
+@pytest.mark.parametrize(
+    ("files", "pytest_command"),
+    [
+        (
+            get_functional2_lang_files(
+                {
+                    "functional2": {
+                        "lang": {
+                            "toml_unused": {
+                                "in.nix": File("{}"),
+                                "in-1.nix": File("{}"),
+                                "eval-fail.err.exp": File(""),
+                                "eval-okay.out.exp": AssetSymlink(
+                                    "assets/test_lang_infra/runner_eo.out.exp"
+                                ),
+                                "test.toml": File(
+                                    dedent("""
+                                    [[test]]
+                                    runner = "eval-okay"
+                                    """)
+                                ),
+                            }
+                        }
+                    }
+                }
+            ),
+            [],
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.usefixtures("files")
+def test_toml_throws_unused_files(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    out = res.stdout_plain
+    assert "test_invalid_configuration[toml_unused-reasons0]" in out
+    assert "the following files weren't referenced: {'in-1.nix', 'eval-fail.err.exp'}" in out
+
+
 @pytest.mark.parametrize(
     ("files", "pytest_command"),
     [