Test coverage for native code WIP

Part of: https://git.lix.systems/lix-project/lix/issues/1186

tests/coverage/coverage-report.py

@@ -0,0 +1,135 @@
+"""
+Generates a report of code coverage using llvm's line-based coverage tool.
+
+This merges together/indexes all the profraw files: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports
+
+Then it generates reports.
+TODO(review): should this maybe be two separate things? idk!
+"""
+
+import sys
+from pathlib import Path
+from dataclasses import dataclass
+import glob
+import tempfile
+import subprocess
+import shlex
+import logging
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+fmt = logging.Formatter(
+    "{asctime} {levelname} {name}: {message}", datefmt="%b %d %H:%M:%S", style="{"
+)
+hand = logging.StreamHandler()
+hand.setFormatter(fmt)
+log.addHandler(hand)
+
+
+def run(args, *rest, check: bool = True, **kwargs):
+    # stringify all path values
+    args = [str(a) for a in args]
+
+    logging.info("Run: %s", shlex.join(args))
+    return subprocess.run(args, *rest, check=check, **kwargs)
+
+
+@dataclass
+class LLVMCovArgs:
+    llvm_cov: Path
+    objects: list[str]
+    source_root: Path
+    profdata: Path
+
+    def to_opts(self) -> list[str]:
+        args = [self.objects[0]]
+        for obj in self.objects:
+            args.extend(["-object", obj])
+
+        args.extend([f"-compilation-dir={self.source_root}", f"-instr-profile={self.profdata}"])
+
+        return args
+
+    def show_html(self, out_dir: Path, *args):
+        run(
+            [
+                self.llvm_cov,
+                "show",
+                *self.to_opts(),
+                "-format=html",
+                f"-output-dir={out_dir}",
+                # by default, it doesn't show coverage of particular
+                # instantiations of template functions, but let's turn it on for
+                # fun!
+                "-show-instantiation-summary",
+                *args,
+            ]
+        )
+
+    def export_lcov(self, out_file: Path, *args):
+        with out_file.open("w") as h:
+            run([self.llvm_cov, "export", *self.to_opts(), "-format=lcov"], stdout=h)
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--profraw-dir", type=Path, help="Directory of the .profraw files to ingest"
+    )
+    parser.add_argument(
+        "--out-dir", type=Path, help="Output directory for reports and intermediates (indexes)"
+    )
+    parser.add_argument(
+        "--source-root", type=Path, help="Source root, given to llvm-cov as -compilation-dir"
+    )
+    parser.add_argument("--llvm-cov", type=Path, help="llvm-cov executable")
+    parser.add_argument("--llvm-profdata", type=Path, help="llvm-profdata executable")
+    parser.add_argument("objects", nargs="+")
+    args = parser.parse_args()
+
+    profraw_dir: Path = args.profraw_dir
+    out_dir: Path = args.out_dir
+    source_root: Path = args.source_root
+    llvm_profdata: Path = args.llvm_profdata
+    llvm_cov: Path = args.llvm_cov
+    objects: list[str] = args.objects
+
+    profraw_dir.mkdir(parents=True, exist_ok=True)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    all_profraws = sorted(glob.glob(str(profraw_dir / "**/*.profraw"), recursive=True))
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+
+        inputs_file = tmp / "inputs"
+        inputs_file.write_text("\n".join(all_profraws) + "\n")
+
+        profdata = out_dir / "merged.profdata"
+
+        run(
+            [
+                llvm_profdata,
+                "merge",
+                # ostensibly faster or smaller according to LLVM docs
+                "-sparse",
+                # TODO: '--failure-mode=warn' ?
+                "-o",
+                profdata,
+                f"--input-files={inputs_file}",
+            ]
+        )
+
+        llvm_cov_args = LLVMCovArgs(
+            llvm_cov=llvm_cov, objects=objects, source_root=source_root, profdata=profdata
+        )
+        llvm_cov_args.show_html(out_dir=out_dir)
+        llvm_cov_args.export_lcov(out_file=out_dir / "coverage.lcov")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())