From c76cf496293602e55ff3aae5bda3e422ebbf4f10 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 19 Apr 2026 17:50:44 +0000 Subject: [PATCH] Test coverage for native code WIP Part of: https://git.lix.systems/lix-project/lix/issues/1186 --- doc/manual/src/contributing/hacking.md | 12 ++ meson.build | 25 ++++ meson.options | 4 + tests/coverage/args/meson.build | 36 +++++ tests/coverage/coverage-report.py | 135 ++++++++++++++++++ tests/coverage/meson.build | 12 ++ tests/functional/meson.build | 2 +- .../repl_characterization/meson.build | 2 +- tests/functional2/meson.build | 4 + tests/unit/meson.build | 10 +- 10 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 tests/coverage/args/meson.build create mode 100644 tests/coverage/coverage-report.py create mode 100644 tests/coverage/meson.build diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 3b5b63a29..0da22103d 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -444,6 +444,18 @@ You can build it yourself: # xdg-open ./result/coverage/index.html ``` +Or, in a dev shell, set `-Dcoverage=true` when running `meson setup`. +Coverage data goes into `build/profraw` when you run executables in the dev shell. +Then, run `ninja -C build coverage-report` to produce an HTML report of coverage in `build/coverage/index.html` alongside a LLVM `.lcov` file. + +> [!NOTE] +> We use the [llvm source-based coverage], which has better precision than using clang with gcov, which is debuginfo based (but likely worse performance, which is fine). +> +> It should be noted that Meson [allegedly has coverage support][meson-coverage], but it only supports gcov-style coverage, so we don't use it. + +[llvm source-based coverage]: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +[meson-coverage]: https://mesonbuild.com/Unit-tests.html#coverage + Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)). ## Add a release note {#release-notes} diff --git a/meson.build b/meson.build index fc2e348ea..64841427a 100644 --- a/meson.build +++ b/meson.build @@ -688,6 +688,11 @@ if cxx.get_id() in ['clang', 'gcc'] language : 'cpp', ) endif +add_project_arguments( + # likewise for rust + '--remap-path-prefix=../lix=lix', + language : 'rust', +) if is_darwin fs.copyfile( @@ -713,10 +718,24 @@ capnpc_wrapper = custom_target( output : 'capnpc_wrapper', ) +coverage_test_env = {} +coverage = get_option('coverage') +if coverage + if cxx.get_id() != 'clang' + error('-Dcoverage=true is llvm-only') + endif + # not necessarily just for our own tests, so we don't gate on tests being + # enabled + subdir('tests/coverage/args') +endif + + subdir('lix') subdir('scripts') subdir('misc') +coverage_objects = [nix, liblix_all] + if enable_docs subdir('doc/manual') endif @@ -734,3 +753,9 @@ endif subdir('meson/clang-tidy') subproject('nix-eval-jobs', required : enable_nix_eval_jobs) + +if coverage + # targets defined by coverage + subdir('tests/coverage') +endif + diff --git a/meson.options b/meson.options index 28fa7c85a..391282565 100644 --- a/meson.options +++ b/meson.options @@ -108,3 +108,7 @@ option('disable-fibers', type : 'boolean', value : false, option('builtin-dep-closure', type : 'array', description : 'dependency closure used for builtin builder sandboxes. the install paths are included automatically.', ) + +option('coverage', type : 'boolean', + description : 'Use LLVM\'s source-based coverage while building and testing Lix' +) diff --git a/tests/coverage/args/meson.build b/tests/coverage/args/meson.build new file mode 100644 index 000000000..be18ec01c --- /dev/null +++ b/tests/coverage/args/meson.build @@ -0,0 +1,36 @@ +# Early initialization of LLVM source-based coverage. +# +# Needs to run before defining any targets. + +add_project_arguments( + '-fprofile-instr-generate', + '-fcoverage-mapping', + # TODO: -mllvm -runtime-counter-relocation may fix problems with tests with + # nix run/fmt/etc that execvp, bypassing atexit. need to confirm that's real. + language : 'cpp', +) +# N.B. This is a link argument because it needs to link the LLVM profiling runtime, I believe. +add_project_link_arguments( + '-fprofile-instr-generate', + language: 'cpp', +) + +add_project_arguments( + '-Cinstrument-coverage', + language : 'rust', +) + +coverage_profraw_dir = meson.project_build_root() / 'profraw' +run_command('mkdir', '-p', coverage_profraw_dir, check : true) +coverage_test_env = { + # TODO: may need %c, but that may only work if you set runtime counter relocation, need to find that out + # + # See: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program + # %20m -> use 20 raw profiles and merge at runtime, so that our large number of + # invocations of lix in the test suite don't create unreasonable numbers of + # files. This, I think, also limits the test concurrency. + 'LLVM_PROFILE_FILE': coverage_profraw_dir / '%20m.profraw' +} + +llvm_profdata = find_program('llvm-profdata', required : true) +llvm_cov = find_program('llvm-cov', required : true) diff --git a/tests/coverage/coverage-report.py b/tests/coverage/coverage-report.py new file mode 100644 index 000000000..97d5f8568 --- /dev/null +++ b/tests/coverage/coverage-report.py @@ -0,0 +1,135 @@ +""" +Generates a report of code coverage using llvm's line-based coverage tool. + +This merges together/indexes all the profraw files: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports + +Then it generates reports. +TODO(review): should this maybe be two separate things? idk! +""" + +import sys +from pathlib import Path +from dataclasses import dataclass +import glob +import tempfile +import subprocess +import shlex +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) +fmt = logging.Formatter( + "{asctime} {levelname} {name}: {message}", datefmt="%b %d %H:%M:%S", style="{" +) +hand = logging.StreamHandler() +hand.setFormatter(fmt) +log.addHandler(hand) + + +def run(args, *rest, check: bool = True, **kwargs): + # stringify all path values + args = [str(a) for a in args] + + logging.info("Run: %s", shlex.join(args)) + return subprocess.run(args, *rest, check=check, **kwargs) + + +@dataclass +class LLVMCovArgs: + llvm_cov: Path + objects: list[str] + source_root: Path + profdata: Path + + def to_opts(self) -> list[str]: + args = [self.objects[0]] + for obj in self.objects: + args.extend(["-object", obj]) + + args.extend([f"-compilation-dir={self.source_root}", f"-instr-profile={self.profdata}"]) + + return args + + def show_html(self, out_dir: Path, *args): + run( + [ + self.llvm_cov, + "show", + *self.to_opts(), + "-format=html", + f"-output-dir={out_dir}", + # by default, it doesn't show coverage of particular + # instantiations of template functions, but let's turn it on for + # fun! + "-show-instantiation-summary", + *args, + ] + ) + + def export_lcov(self, out_file: Path, *args): + with out_file.open("w") as h: + run([self.llvm_cov, "export", *self.to_opts(), "-format=lcov"], stdout=h) + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profraw-dir", type=Path, help="Directory of the .profraw files to ingest" + ) + parser.add_argument( + "--out-dir", type=Path, help="Output directory for reports and intermediates (indexes)" + ) + parser.add_argument( + "--source-root", type=Path, help="Source root, given to llvm-cov as -compilation-dir" + ) + parser.add_argument("--llvm-cov", type=Path, help="llvm-cov executable") + parser.add_argument("--llvm-profdata", type=Path, help="llvm-profdata executable") + parser.add_argument("objects", nargs="+") + args = parser.parse_args() + + profraw_dir: Path = args.profraw_dir + out_dir: Path = args.out_dir + source_root: Path = args.source_root + llvm_profdata: Path = args.llvm_profdata + llvm_cov: Path = args.llvm_cov + objects: list[str] = args.objects + + profraw_dir.mkdir(parents=True, exist_ok=True) + out_dir.mkdir(parents=True, exist_ok=True) + + all_profraws = sorted(glob.glob(str(profraw_dir / "**/*.profraw"), recursive=True)) + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + + inputs_file = tmp / "inputs" + inputs_file.write_text("\n".join(all_profraws) + "\n") + + profdata = out_dir / "merged.profdata" + + run( + [ + llvm_profdata, + "merge", + # ostensibly faster or smaller according to LLVM docs + "-sparse", + # TODO: '--failure-mode=warn' ? + "-o", + profdata, + f"--input-files={inputs_file}", + ] + ) + + llvm_cov_args = LLVMCovArgs( + llvm_cov=llvm_cov, objects=objects, source_root=source_root, profdata=profdata + ) + llvm_cov_args.show_html(out_dir=out_dir) + llvm_cov_args.export_lcov(out_file=out_dir / "coverage.lcov") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/coverage/meson.build b/tests/coverage/meson.build new file mode 100644 index 000000000..37330dcc6 --- /dev/null +++ b/tests/coverage/meson.build @@ -0,0 +1,12 @@ +# LLVM source-based coverage for Lix +run_target('coverage-report', + command : [ + python, meson.project_source_root() / 'tests/coverage/coverage-report.py', + '--profraw-dir', coverage_profraw_dir, + '--out-dir', meson.project_build_root() / 'meson-logs/coverage', + '--source-root', meson.project_source_root(), + '--llvm-profdata', llvm_profdata.full_path(), + '--llvm-cov', llvm_cov.full_path(), + coverage_objects, + ], +) diff --git a/tests/functional/meson.build b/tests/functional/meson.build index f521510ec..bf2f20779 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -154,7 +154,7 @@ foreach script : functional_tests_scripts suite : 'installcheck', env : { 'MESON_BUILD_ROOT': meson.project_build_root(), - }, + } + coverage_test_env, # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine # this can easily drive them to failure. give them more time, 5min rather than 30sec timeout : 300, diff --git a/tests/functional/repl_characterization/meson.build b/tests/functional/repl_characterization/meson.build index dfafe59b8..d5288d59e 100644 --- a/tests/functional/repl_characterization/meson.build +++ b/tests/functional/repl_characterization/meson.build @@ -36,6 +36,6 @@ test( env : { '_NIX_TEST_UNIT_DATA': meson.current_build_dir() / 'data', 'MESON_BUILD_ROOT': meson.project_build_root(), - }, + } + coverage_test_env, suite : 'installcheck', ) diff --git a/tests/functional2/meson.build b/tests/functional2/meson.build index cf5519976..e2beef703 100644 --- a/tests/functional2/meson.build +++ b/tests/functional2/meson.build @@ -23,6 +23,10 @@ if build_test_env != '' endif functional2_env.set('system', host_system) +foreach name, val : coverage_test_env + functional2_env.set(name, val) +endforeach + test( 'functional2', bash, diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 1cb2ce143..3e89020df 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -17,7 +17,7 @@ default_test_env = { 'NIX_CONF_DIR': '/var/empty', # Prevent loading user configuration files in tests 'NIX_USER_CONF_FILES': '', -} +} + coverage_test_env libutil_test_support_sources = files( 'libutil-support/tests/cli-literate-parser.cc', @@ -296,3 +296,11 @@ test( suite : 'check', protocol : 'gtest', ) + +coverage_objects += [ + libutil_tester, + libstore_tester, + libexpr_tester, + libcmd_tester, + libmain_tester, +]