Test coverage for native code WIP

Part of: https://git.lix.systems/lix-project/lix/issues/1186

tests/coverage/args/meson.build

@@ -0,0 +1,36 @@
+# Early initialization of LLVM source-based coverage.
+#
+# Needs to run before defining any targets.
+
+add_project_arguments(
+  '-fprofile-instr-generate',
+  '-fcoverage-mapping',
+  # TODO: -mllvm -runtime-counter-relocation may fix problems with tests with
+  # nix run/fmt/etc that execvp, bypassing atexit. need to confirm that's real.
+  language : 'cpp',
+)
+# N.B. This is a link argument because it needs to link the LLVM profiling runtime, I believe.
+add_project_link_arguments(
+  '-fprofile-instr-generate',
+  language: 'cpp',
+)
+
+add_project_arguments(
+  '-Cinstrument-coverage',
+  language : 'rust',
+)
+
+coverage_profraw_dir = meson.project_build_root() / 'profraw'
+run_command('mkdir', '-p', coverage_profraw_dir, check : true)
+coverage_test_env = {
+  # TODO: may need %c, but that may only work if you set runtime counter relocation, need to find that out
+  #
+  # See: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program
+  # %20m -> use 20 raw profiles and merge at runtime, so that our large number of
+  # invocations of lix in the test suite don't create unreasonable numbers of
+  # files. This, I think, also limits the test concurrency.
+  'LLVM_PROFILE_FILE': coverage_profraw_dir / '%20m.profraw'
+}
+
+llvm_profdata = find_program('llvm-profdata', required : true)
+llvm_cov = find_program('llvm-cov', required : true)

tests/coverage/coverage-report.py

@@ -0,0 +1,135 @@
+"""
+Generates a report of code coverage using llvm's line-based coverage tool.
+
+This merges together/indexes all the profraw files: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports
+
+Then it generates reports.
+TODO(review): should this maybe be two separate things? idk!
+"""
+
+import sys
+from pathlib import Path
+from dataclasses import dataclass
+import glob
+import tempfile
+import subprocess
+import shlex
+import logging
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+fmt = logging.Formatter(
+    "{asctime} {levelname} {name}: {message}", datefmt="%b %d %H:%M:%S", style="{"
+)
+hand = logging.StreamHandler()
+hand.setFormatter(fmt)
+log.addHandler(hand)
+
+
+def run(args, *rest, check: bool = True, **kwargs):
+    # stringify all path values
+    args = [str(a) for a in args]
+
+    logging.info("Run: %s", shlex.join(args))
+    return subprocess.run(args, *rest, check=check, **kwargs)
+
+
+@dataclass
+class LLVMCovArgs:
+    llvm_cov: Path
+    objects: list[str]
+    source_root: Path
+    profdata: Path
+
+    def to_opts(self) -> list[str]:
+        args = [self.objects[0]]
+        for obj in self.objects:
+            args.extend(["-object", obj])
+
+        args.extend([f"-compilation-dir={self.source_root}", f"-instr-profile={self.profdata}"])
+
+        return args
+
+    def show_html(self, out_dir: Path, *args):
+        run(
+            [
+                self.llvm_cov,
+                "show",
+                *self.to_opts(),
+                "-format=html",
+                f"-output-dir={out_dir}",
+                # by default, it doesn't show coverage of particular
+                # instantiations of template functions, but let's turn it on for
+                # fun!
+                "-show-instantiation-summary",
+                *args,
+            ]
+        )
+
+    def export_lcov(self, out_file: Path, *args):
+        with out_file.open("w") as h:
+            run([self.llvm_cov, "export", *self.to_opts(), "-format=lcov"], stdout=h)
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--profraw-dir", type=Path, help="Directory of the .profraw files to ingest"
+    )
+    parser.add_argument(
+        "--out-dir", type=Path, help="Output directory for reports and intermediates (indexes)"
+    )
+    parser.add_argument(
+        "--source-root", type=Path, help="Source root, given to llvm-cov as -compilation-dir"
+    )
+    parser.add_argument("--llvm-cov", type=Path, help="llvm-cov executable")
+    parser.add_argument("--llvm-profdata", type=Path, help="llvm-profdata executable")
+    parser.add_argument("objects", nargs="+")
+    args = parser.parse_args()
+
+    profraw_dir: Path = args.profraw_dir
+    out_dir: Path = args.out_dir
+    source_root: Path = args.source_root
+    llvm_profdata: Path = args.llvm_profdata
+    llvm_cov: Path = args.llvm_cov
+    objects: list[str] = args.objects
+
+    profraw_dir.mkdir(parents=True, exist_ok=True)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    all_profraws = sorted(glob.glob(str(profraw_dir / "**/*.profraw"), recursive=True))
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+
+        inputs_file = tmp / "inputs"
+        inputs_file.write_text("\n".join(all_profraws) + "\n")
+
+        profdata = out_dir / "merged.profdata"
+
+        run(
+            [
+                llvm_profdata,
+                "merge",
+                # ostensibly faster or smaller according to LLVM docs
+                "-sparse",
+                # TODO: '--failure-mode=warn' ?
+                "-o",
+                profdata,
+                f"--input-files={inputs_file}",
+            ]
+        )
+
+        llvm_cov_args = LLVMCovArgs(
+            llvm_cov=llvm_cov, objects=objects, source_root=source_root, profdata=profdata
+        )
+        llvm_cov_args.show_html(out_dir=out_dir)
+        llvm_cov_args.export_lcov(out_file=out_dir / "coverage.lcov")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/coverage/meson.build

@@ -0,0 +1,12 @@
+# LLVM source-based coverage for Lix
+run_target('coverage-report',
+  command : [
+    python, meson.project_source_root() / 'tests/coverage/coverage-report.py',
+    '--profraw-dir', coverage_profraw_dir,
+    '--out-dir', meson.project_build_root() / 'meson-logs/coverage',
+    '--source-root', meson.project_source_root(),
+    '--llvm-profdata', llvm_profdata.full_path(),
+    '--llvm-cov', llvm_cov.full_path(),
+    coverage_objects,
+  ],
+)