From 7553d0a983f2b35b563601b620f90d4655341252 Mon Sep 17 00:00:00 2001
From: "Commentator2.0" <commentator2.0@crystal-cavern.systems>
Date: Mon, 18 Aug 2025 20:23:18 +0200
Subject: [PATCH] tests/functional2/lang: don't throw unused file errors on
 invalid configurations

Currenlty, when a test group is invalid already, we also throw unsued
file errors.
This leads to clutter as more often than not, the unused files are
caused by an invalid configuration, making the debug stack bigger
without reason.

With this commit the behavior is changed to only error about unused
files, when no other configuration issues were found

Change-Id: I92a819753f13b8ed5a07dae53ecaee5d84b5ce64
---
 tests/functional2/lang/lang_util.py       | 11 +++-
 tests/functional2/lang/test_lang_infra.py | 61 +++++++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/tests/functional2/lang/lang_util.py b/tests/functional2/lang/lang_util.py
index 5539a562c..66a5f6718 100644
--- a/tests/functional2/lang/lang_util.py
+++ b/tests/functional2/lang/lang_util.py
@@ -323,7 +323,10 @@ def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[Invalid
         for t in new_tests:
             unused_files -= {t.in_file_name, f"{t.test_name}.out.exp", f"{t.test_name}.err.exp"}
 
-    if len(unused_files) > 0:
+    # Only throw issues about unused files when the group is otherwise valid
+    # this is done to avoid having unused files showing up due to invalid configurations,
+    # which should be fixed first and the unused files would just clutter the error messages in that case
+    if len(unused_files) > 0 and not invalid_tests:
         invalid_tests.append(
             InvalidLangTest(
                 parent_name, [f"the following files weren't referenced: {unused_files!r}"]
@@ -370,7 +373,11 @@ def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[Inva
                 LangTest(runner_name, folder.name, runner, InFile(f"in{suffix}.nix", suffix))
             )
             unused -= {file.name, f"in{suffix or ''}.nix"}
-    if len(unused) > 0:
+
+    # Only throw issues about unused files when the group is otherwise valid
+    # this is done to avoid having unused files showing up due to invalid configurations,
+    # which should be fixed first and the unused files would just clutter the error messages in that case
+    if len(unused) > 0 and not invalid_tests:
         invalid_tests.append(
             InvalidLangTest(parent_name, [f"the following files weren't referenced: {unused!r}"])
         )
diff --git a/tests/functional2/lang/test_lang_infra.py b/tests/functional2/lang/test_lang_infra.py
index 778ba26d7..bf0a2816a 100644
--- a/tests/functional2/lang/test_lang_infra.py
+++ b/tests/functional2/lang/test_lang_infra.py
@@ -471,6 +471,67 @@ def test_toml_throws_unused_files(
     )
 
 
+@pytest.mark.parametrize("pytest_command", [[]], indirect=True)
+@with_files(
+    get_functional2_lang_files(
+        {
+            "functional2": {
+                "lang": {
+                    "toml_unused": {
+                        "in.nix": File("{}"),
+                        "in-1.nix": File("{}"),
+                        "eval-fail.err.exp": File(""),
+                        "eval-okay.out.exp": AssetSymlink(
+                            "assets/test_lang_infra/runner_eo.out.exp"
+                        ),
+                        "test.toml": File(
+                            dedent("""
+                                    [[test]]
+                                    runner = "eval-okay"
+                                    [[test]]
+                                    runner = "bad-name"
+                                    """)
+                        ),
+                    }
+                }
+            }
+        }
+    )
+)
+def test_toml_no_unused_on_invalid_config(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    log = res.stdout_plain
+    assert "test_invalid_configuration[toml_unused:bad-name-reasons0]" in log
+    assert "invalid runner name:" in log
+    assert "the following files weren't referenced" not in log
+
+
+@pytest.mark.parametrize("pytest_command", [[]], indirect=True)
+@with_files(
+    get_functional2_lang_files(
+        {
+            "functional2": {
+                "lang": {
+                    "generic_unused": {
+                        "in.nix": File("{}"),
+                        "in-1.nix": File("{}"),
+                        "shrimpy-okay.out.exp": AssetSymlink(
+                            "assets/test_lang_infra/runner_eo.out.exp"
+                        ),
+                    }
+                }
+            }
+        }
+    )
+)
+def test_generic_no_unused_on_invalid_config(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    out = res.stdout_plain
+    assert "test_invalid_configuration[generic_unused:shrimpy-okay-reasons0]" in out
+    assert "invalid runner name:" in out
+    assert "the following files weren't referenced" not in out
+
+
 @pytest.mark.parametrize("pytest_command", [[]], indirect=True)
 @with_files(
     get_functional2_lang_files(