diff --git a/tests/functional2/lang/lang_util.py b/tests/functional2/lang/lang_util.py
index 5539a562c..66a5f6718 100644
--- a/tests/functional2/lang/lang_util.py
+++ b/tests/functional2/lang/lang_util.py
@@ -323,7 +323,10 @@ def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[Invalid
         for t in new_tests:
             unused_files -= {t.in_file_name, f"{t.test_name}.out.exp", f"{t.test_name}.err.exp"}
 
-    if len(unused_files) > 0:
+    # Only throw issues about unused files when the group is otherwise valid
+    # this is done to avoid having unused files showing up due to invalid configurations,
+    # which should be fixed first and the unused files would just clutter the error messages in that case
+    if len(unused_files) > 0 and not invalid_tests:
         invalid_tests.append(
             InvalidLangTest(
                 parent_name, [f"the following files weren't referenced: {unused_files!r}"]
@@ -370,7 +373,11 @@ def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[Inva
                 LangTest(runner_name, folder.name, runner, InFile(f"in{suffix}.nix", suffix))
             )
             unused -= {file.name, f"in{suffix or ''}.nix"}
-    if len(unused) > 0:
+
+    # Only throw issues about unused files when the group is otherwise valid
+    # this is done to avoid having unused files showing up due to invalid configurations,
+    # which should be fixed first and the unused files would just clutter the error messages in that case
+    if len(unused) > 0 and not invalid_tests:
         invalid_tests.append(
             InvalidLangTest(parent_name, [f"the following files weren't referenced: {unused!r}"])
         )
diff --git a/tests/functional2/lang/test_lang_infra.py b/tests/functional2/lang/test_lang_infra.py
index 778ba26d7..bf0a2816a 100644
--- a/tests/functional2/lang/test_lang_infra.py
+++ b/tests/functional2/lang/test_lang_infra.py
@@ -471,6 +471,67 @@ def test_toml_throws_unused_files(
     )
 
 
+@pytest.mark.parametrize("pytest_command", [[]], indirect=True)
+@with_files(
+    get_functional2_lang_files(
+        {
+            "functional2": {
+                "lang": {
+                    "toml_unused": {
+                        "in.nix": File("{}"),
+                        "in-1.nix": File("{}"),
+                        "eval-fail.err.exp": File(""),
+                        "eval-okay.out.exp": AssetSymlink(
+                            "assets/test_lang_infra/runner_eo.out.exp"
+                        ),
+                        "test.toml": File(
+                            dedent("""
+                                    [[test]]
+                                    runner = "eval-okay"
+                                    [[test]]
+                                    runner = "bad-name"
+                                    """)
+                        ),
+                    }
+                }
+            }
+        }
+    )
+)
+def test_toml_no_unused_on_invalid_config(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    log = res.stdout_plain
+    assert "test_invalid_configuration[toml_unused:bad-name-reasons0]" in log
+    assert "invalid runner name:" in log
+    assert "the following files weren't referenced" not in log
+
+
+@pytest.mark.parametrize("pytest_command", [[]], indirect=True)
+@with_files(
+    get_functional2_lang_files(
+        {
+            "functional2": {
+                "lang": {
+                    "generic_unused": {
+                        "in.nix": File("{}"),
+                        "in-1.nix": File("{}"),
+                        "shrimpy-okay.out.exp": AssetSymlink(
+                            "assets/test_lang_infra/runner_eo.out.exp"
+                        ),
+                    }
+                }
+            }
+        }
+    )
+)
+def test_generic_no_unused_on_invalid_config(pytest_command: Command):
+    res = pytest_command.run().expect(1)
+    out = res.stdout_plain
+    assert "test_invalid_configuration[generic_unused:shrimpy-okay-reasons0]" in out
+    assert "invalid runner name:" in out
+    assert "the following files weren't referenced" not in out
+
+
 @pytest.mark.parametrize("pytest_command", [[]], indirect=True)
 @with_files(
     get_functional2_lang_files(