Test coverage for native code WIP

Part of: https://git.lix.systems/lix-project/lix/issues/1186

doc/manual/src/contributing/hacking.md

@@ -444,6 +444,18 @@ You can build it yourself:
 # xdg-open ./result/coverage/index.html
 ```
 
+Or, in a dev shell, set `-Dcoverage=true` when running `meson setup`.
+Coverage data goes into `build/profraw` when you run executables in the dev shell.
+Then, run `ninja -C build coverage-report` to produce an HTML report of coverage in `build/coverage/index.html` alongside a LLVM `.lcov` file.
+
+> [!NOTE]
+> We use the [llvm source-based coverage], which has better precision than using clang with gcov, which is debuginfo based (but likely worse performance, which is fine).
+>
+> It should be noted that Meson [allegedly has coverage support][meson-coverage], but it only supports gcov-style coverage, so we don't use it.
+
+[llvm source-based coverage]: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+[meson-coverage]: https://mesonbuild.com/Unit-tests.html#coverage
+
 Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)).
 
 ## Add a release note {#release-notes}

meson.build

@@ -688,6 +688,11 @@ if cxx.get_id() in ['clang', 'gcc']
     language : 'cpp',
   )
 endif
+add_project_arguments(
+  # likewise for rust
+  '--remap-path-prefix=../lix=lix',
+  language : 'rust',
+)
 
 if is_darwin
   fs.copyfile(
@@ -713,10 +718,24 @@ capnpc_wrapper = custom_target(
   output : 'capnpc_wrapper',
 )
 
+coverage_test_env = {}
+coverage = get_option('coverage')
+if coverage
+  if cxx.get_id() != 'clang'
+    error('-Dcoverage=true is llvm-only')
+  endif
+  # not necessarily just for our own tests, so we don't gate on tests being
+  # enabled
+  subdir('tests/coverage/args')
+endif
+
+
 subdir('lix')
 subdir('scripts')
 subdir('misc')
 
+coverage_objects = [nix, liblix_all]
+
 if enable_docs
   subdir('doc/manual')
 endif
@@ -734,3 +753,9 @@ endif
 subdir('meson/clang-tidy')
 
 subproject('nix-eval-jobs', required : enable_nix_eval_jobs)
+
+if coverage
+  # targets defined by coverage
+  subdir('tests/coverage')
+endif
+

meson.options

@@ -108,3 +108,7 @@ option('disable-fibers', type : 'boolean', value : false,
 option('builtin-dep-closure', type : 'array',
   description : 'dependency closure used for builtin builder sandboxes. the install paths are included automatically.',
 )
+
+option('coverage', type : 'boolean',
+  description : 'Use LLVM\'s source-based coverage while building and testing Lix'
+)

tests/coverage/args/meson.build

@@ -0,0 +1,36 @@
+# Early initialization of LLVM source-based coverage.
+#
+# Needs to run before defining any targets.
+
+add_project_arguments(
+  '-fprofile-instr-generate',
+  '-fcoverage-mapping',
+  # TODO: -mllvm -runtime-counter-relocation may fix problems with tests with
+  # nix run/fmt/etc that execvp, bypassing atexit. need to confirm that's real.
+  language : 'cpp',
+)
+# N.B. This is a link argument because it needs to link the LLVM profiling runtime, I believe.
+add_project_link_arguments(
+  '-fprofile-instr-generate',
+  language: 'cpp',
+)
+
+add_project_arguments(
+  '-Cinstrument-coverage',
+  language : 'rust',
+)
+
+coverage_profraw_dir = meson.project_build_root() / 'profraw'
+run_command('mkdir', '-p', coverage_profraw_dir, check : true)
+coverage_test_env = {
+  # TODO: may need %c, but that may only work if you set runtime counter relocation, need to find that out
+  #
+  # See: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program
+  # %20m -> use 20 raw profiles and merge at runtime, so that our large number of
+  # invocations of lix in the test suite don't create unreasonable numbers of
+  # files. This, I think, also limits the test concurrency.
+  'LLVM_PROFILE_FILE': coverage_profraw_dir / '%20m.profraw'
+}
+
+llvm_profdata = find_program('llvm-profdata', required : true)
+llvm_cov = find_program('llvm-cov', required : true)

tests/coverage/coverage-report.py

@@ -0,0 +1,135 @@
+"""
+Generates a report of code coverage using llvm's line-based coverage tool.
+
+This merges together/indexes all the profraw files: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports
+
+Then it generates reports.
+TODO(review): should this maybe be two separate things? idk!
+"""
+
+import sys
+from pathlib import Path
+from dataclasses import dataclass
+import glob
+import tempfile
+import subprocess
+import shlex
+import logging
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+fmt = logging.Formatter(
+    "{asctime} {levelname} {name}: {message}", datefmt="%b %d %H:%M:%S", style="{"
+)
+hand = logging.StreamHandler()
+hand.setFormatter(fmt)
+log.addHandler(hand)
+
+
+def run(args, *rest, check: bool = True, **kwargs):
+    # stringify all path values
+    args = [str(a) for a in args]
+
+    logging.info("Run: %s", shlex.join(args))
+    return subprocess.run(args, *rest, check=check, **kwargs)
+
+
+@dataclass
+class LLVMCovArgs:
+    llvm_cov: Path
+    objects: list[str]
+    source_root: Path
+    profdata: Path
+
+    def to_opts(self) -> list[str]:
+        args = [self.objects[0]]
+        for obj in self.objects:
+            args.extend(["-object", obj])
+
+        args.extend([f"-compilation-dir={self.source_root}", f"-instr-profile={self.profdata}"])
+
+        return args
+
+    def show_html(self, out_dir: Path, *args):
+        run(
+            [
+                self.llvm_cov,
+                "show",
+                *self.to_opts(),
+                "-format=html",
+                f"-output-dir={out_dir}",
+                # by default, it doesn't show coverage of particular
+                # instantiations of template functions, but let's turn it on for
+                # fun!
+                "-show-instantiation-summary",
+                *args,
+            ]
+        )
+
+    def export_lcov(self, out_file: Path, *args):
+        with out_file.open("w") as h:
+            run([self.llvm_cov, "export", *self.to_opts(), "-format=lcov"], stdout=h)
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--profraw-dir", type=Path, help="Directory of the .profraw files to ingest"
+    )
+    parser.add_argument(
+        "--out-dir", type=Path, help="Output directory for reports and intermediates (indexes)"
+    )
+    parser.add_argument(
+        "--source-root", type=Path, help="Source root, given to llvm-cov as -compilation-dir"
+    )
+    parser.add_argument("--llvm-cov", type=Path, help="llvm-cov executable")
+    parser.add_argument("--llvm-profdata", type=Path, help="llvm-profdata executable")
+    parser.add_argument("objects", nargs="+")
+    args = parser.parse_args()
+
+    profraw_dir: Path = args.profraw_dir
+    out_dir: Path = args.out_dir
+    source_root: Path = args.source_root
+    llvm_profdata: Path = args.llvm_profdata
+    llvm_cov: Path = args.llvm_cov
+    objects: list[str] = args.objects
+
+    profraw_dir.mkdir(parents=True, exist_ok=True)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    all_profraws = sorted(glob.glob(str(profraw_dir / "**/*.profraw"), recursive=True))
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+
+        inputs_file = tmp / "inputs"
+        inputs_file.write_text("\n".join(all_profraws) + "\n")
+
+        profdata = out_dir / "merged.profdata"
+
+        run(
+            [
+                llvm_profdata,
+                "merge",
+                # ostensibly faster or smaller according to LLVM docs
+                "-sparse",
+                # TODO: '--failure-mode=warn' ?
+                "-o",
+                profdata,
+                f"--input-files={inputs_file}",
+            ]
+        )
+
+        llvm_cov_args = LLVMCovArgs(
+            llvm_cov=llvm_cov, objects=objects, source_root=source_root, profdata=profdata
+        )
+        llvm_cov_args.show_html(out_dir=out_dir)
+        llvm_cov_args.export_lcov(out_file=out_dir / "coverage.lcov")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/coverage/meson.build

@@ -0,0 +1,12 @@
+# LLVM source-based coverage for Lix
+run_target('coverage-report',
+  command : [
+    python, meson.project_source_root() / 'tests/coverage/coverage-report.py',
+    '--profraw-dir', coverage_profraw_dir,
+    '--out-dir', meson.project_build_root() / 'meson-logs/coverage',
+    '--source-root', meson.project_source_root(),
+    '--llvm-profdata', llvm_profdata.full_path(),
+    '--llvm-cov', llvm_cov.full_path(),
+    coverage_objects,
+  ],
+)

tests/functional/meson.build

@@ -154,7 +154,7 @@ foreach script : functional_tests_scripts
     suite : 'installcheck',
     env : {
       'MESON_BUILD_ROOT': meson.project_build_root(),
-    },
+    } + coverage_test_env,
     # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine
     # this can easily drive them to failure. give them more time, 5min rather than 30sec
     timeout : 300,

tests/functional/repl_characterization/meson.build

@@ -36,6 +36,6 @@ test(
   env : {
     '_NIX_TEST_UNIT_DATA': meson.current_build_dir() / 'data',
     'MESON_BUILD_ROOT': meson.project_build_root(),
-  },
+  } + coverage_test_env,
   suite : 'installcheck',
 )

tests/functional2/meson.build

@@ -23,6 +23,10 @@ if build_test_env != ''
 endif
 functional2_env.set('system', host_system)
 
+foreach name, val : coverage_test_env
+  functional2_env.set(name, val)
+endforeach
+
 test(
   'functional2',
   bash,

tests/unit/meson.build

@@ -17,7 +17,7 @@ default_test_env = {
   'NIX_CONF_DIR': '/var/empty',
   # Prevent loading user configuration files in tests
   'NIX_USER_CONF_FILES': '',
-}
+} + coverage_test_env
 
 libutil_test_support_sources = files(
   'libutil-support/tests/cli-literate-parser.cc',
@@ -296,3 +296,11 @@ test(
   suite : 'check',
   protocol : 'gtest',
 )
+
+coverage_objects += [
+  libutil_tester,
+  libstore_tester,
+  libexpr_tester,
+  libcmd_tester,
+  libmain_tester,
+]