From cad275307ef3857474adbebe67fe7ec92f86d24e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 23 Mar 2025 17:47:20 -0700 Subject: [PATCH] observability: add the first USDT probe USDT probes are statically defined trace points that have nearly zero disabled-probe effect, i.e. we can put them in hot paths. The use case for these is both similar and dissimilar to Rust tracing: We still need better logging and a better structured rust-tracing looking thing, but probes allow for quite easy programmable interactive tracing in production, which we also care a lot about. This CL comes with a perfunctory trace point in libstore/file-transfer.cc for reading data out of the curl buffer. This was mostly thrown in there so that I could see what the buffer sizes of this were, and maybe be able to instrument the perf of the curl usages in Lix in the future. Fixes: https://git.lix.systems/lix-project/lix/issues/727 Change-Id: I0f5d9912d76bf3d6923bf53ebfd9b8d6c6e70aea --- doc/manual/rl-next/usdt-support.md | 35 ++++++++++++++++++++++++++++++ lix/libstore/filetransfer.cc | 3 +++ lix/libstore/meson.build | 7 ++++++ lix/libstore/trace-probes.d | 9 ++++++++ lix/libutil/meson.build | 1 + lix/libutil/tracepoint.hh | 24 ++++++++++++++++++++ meson.build | 26 ++++++++++++++++++++++ meson.options | 4 ++++ package.nix | 22 +++++++++++++++++++ 9 files changed, 131 insertions(+) create mode 100644 doc/manual/rl-next/usdt-support.md create mode 100644 lix/libstore/trace-probes.d create mode 100644 lix/libutil/tracepoint.hh diff --git a/doc/manual/rl-next/usdt-support.md b/doc/manual/rl-next/usdt-support.md new file mode 100644 index 000000000..e741f7911 --- /dev/null +++ b/doc/manual/rl-next/usdt-support.md @@ -0,0 +1,35 @@ +--- +synopsis: "Add support for eBPF USDT/dtrace probes inside Lix" +issues: [fj#727] +cls: [2884] +category: Features +credits: jade +--- + +eBPF tracers like `bpftrace` and `dtrace` are a group of similar tools for debugging production systems. +User-space statically defined tracing probes (USDT) allow for defining zero or near-zero disabled-probe-effect probes, thus allowing instrumentation of hot paths in production builds. +Lix now has internal support for defining these probes and has shipped its first probe. + +As of this writing it is available by default in the Linux build of Lix. + +To try it out on Linux, you can use the following example command: + +``` +$ sudo bpftrace -l 'usdt:/path/to/liblixstore.so:*:*' +usdt:/path/to/liblixstore.so:lix_store:filetransfer__read + +$ sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }' +Attaching 1 probe... +https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192 +https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192 +https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536 +https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536 +``` + +Note that bpftrace does not offer any way to list the arguments to USDT probes in a human readable form. +To get the probe definitions, see the `*.d` files in the Lix source code, for example, `lix/libstore/trace-probes.d`. + +For more resources on eBPF/bpftrace and dtrace, see: +* The book "BPF Performance Tools" by Brendan Gregg, which discusses bpftrace at length. +* +* [Illumos' dtrace book](https://illumos.org/books/dtrace/preface.html) diff --git a/lix/libstore/filetransfer.cc b/lix/libstore/filetransfer.cc index f94691974..cfc340874 100644 --- a/lix/libstore/filetransfer.cc +++ b/lix/libstore/filetransfer.cc @@ -8,6 +8,7 @@ #include "lix/libutil/signals.hh" #include "lix/libutil/strings.hh" #include "lix/libutil/thread-name.hh" +#include "lix/libutil/tracepoint.hh" #include #include @@ -947,6 +948,8 @@ struct curlFileTransfer : public FileTransfer size_t read(char * data, size_t len) override { + TRACE(DTRACE_PROBE2(lix_store, filetransfer__read, uri.c_str(), len)); + size_t total = 0; while (total < len && awaitData()) { const auto available = std::min(len - total, buffered.size()); diff --git a/lix/libstore/meson.build b/lix/libstore/meson.build index 2b2616d06..27145df8d 100644 --- a/lix/libstore/meson.build +++ b/lix/libstore/meson.build @@ -128,6 +128,12 @@ libstore_settings_headers += custom_target( install_dir : includedir / 'lix/libstore', ) +libstore_extra_objects = [] +if dtrace_feature.enabled() + libstore_settings_headers += dtrace_header_gen.process('trace-probes.d') + libstore_extra_objects += dtrace_object_gen.process('trace-probes.d') +endif + libstore_sources = files( # keep-sorted start 'binary-cache-store.cc', @@ -358,6 +364,7 @@ libstore = library( libstore_sources, libstore_settings_headers, libstore_generated_headers, + libstore_extra_objects, include_directories : [ '../..' ], dependencies : dependencies, cpp_args : cpp_args, diff --git a/lix/libstore/trace-probes.d b/lix/libstore/trace-probes.d new file mode 100644 index 000000000..580b00310 --- /dev/null +++ b/lix/libstore/trace-probes.d @@ -0,0 +1,9 @@ +// I don't know how the rest of these stability attrs work and tbh I don't care +// either; these probes are not stable API and we don't need to be more +// specific about exactly how. +#pragma D attributes Unstable/Unstable/Common provider lix_store provider + +provider lix_store { + /** See filetransfer.cc; this is the consumption side, not the curl/production side. */ + probe filetransfer__read(string url, size_t length); +}; diff --git a/lix/libutil/meson.build b/lix/libutil/meson.build index a78f67e5b..642b87bcc 100644 --- a/lix/libutil/meson.build +++ b/lix/libutil/meson.build @@ -129,6 +129,7 @@ libutil_headers = files( 'thread-name.hh', 'thread-pool.hh', 'topo-sort.hh', + 'tracepoint.hh', 'types.hh', 'unix-domain-socket.hh', 'url-name.hh', diff --git a/lix/libutil/tracepoint.hh b/lix/libutil/tracepoint.hh new file mode 100644 index 000000000..edbbc0c4b --- /dev/null +++ b/lix/libutil/tracepoint.hh @@ -0,0 +1,24 @@ +#pragma once +/** @file USDT based trace points + * + * These can be used with bpftrace or dtrace using their respective USDT trace + * providers. + * + * See the .d files in each library for probe details. + * + * Example: + * + * ``` + * sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }' + * ``` + */ + +#include "lix/config.h" + +// NOTE: glib disables this for the clang static analyzer, idk if we need to also +#if HAVE_DTRACE +#define TRACE(body) body +#include +#else +#define TRACE(body) +#endif diff --git a/meson.build b/meson.build index d66ff1a8c..8a23590e1 100644 --- a/meson.build +++ b/meson.build @@ -402,6 +402,32 @@ endif # FIXME(Qyriad): the autoconf system checks that busybox has the "standalone" feature, indicating # that busybox sh won't run busybox applets as builtins (which would break our sandbox). +# Trace points for dtrace and bpftrace +dtrace_feature = get_option('dtrace-probes') +if dtrace_feature.allowed() + dtrace_exe = find_program('dtrace', native : false, required : false) + dtrace_header_present = cxx.has_header('sys/sdt.h') + dtrace_feature = dtrace_feature.enable_auto_if(dtrace_exe.found() and dtrace_header_present) +endif + +if dtrace_feature.enabled() + dtrace_feature.require(dtrace_exe.found() and dtrace_header_present, + error_message : 'trace probes require both the dtrace command and sys/sdt.h header to be available') + + # NOTE: glib seems to have had to hack some stuff up that we are not hacking + # up. I don't know why. + # https://github.com/GNOME/glib/blob/03f7c1fbf3a3784cb4c3604f83ca3645e9225577/meson.build#L2420-L2434 + dtrace_object_gen = generator(dtrace_exe, + output : '@BASENAME@.o', + arguments : ['-G', '-s', '@INPUT@', '-o', '@OUTPUT@']) + dtrace_header_gen = generator(dtrace_exe, + output : '@BASENAME@.gen.h', + arguments : ['-h', '-s', '@INPUT@', '-o', '@OUTPUT@']) +endif +configdata += { + 'HAVE_DTRACE': dtrace_feature.enabled().to_int(), +} + lsof = find_program('lsof', native : true) # This is how Nix does generated headers... diff --git a/meson.options b/meson.options index 81efba8b9..8d5eed0bc 100644 --- a/meson.options +++ b/meson.options @@ -73,6 +73,10 @@ option('enable-pch-std', type : 'boolean', value : true, description : 'whether to use precompiled headers for C++\'s standard library (breaks clangd if you\'re using GCC)', ) +option('dtrace-probes', type : 'feature', value : 'auto', + description : 'whether to build dtrace/bpftrace compatible USDT probes into Lix' +) + option('lix-clang-tidy-checks-path', type : 'string', value : '', description: 'path to lix-clang-tidy-checks library file, if providing it externally. Uses an internal one if this is not set', ) diff --git a/package.nix b/package.nix index 31fc68145..5a2f0d166 100644 --- a/package.nix +++ b/package.nix @@ -30,6 +30,8 @@ libcpuid, libseccomp, libsodium, + libsystemtap, + linuxPackages, lix-clang-tidy ? null, llvmPackages, lsof, @@ -51,6 +53,7 @@ rustPlatform, rustc, sqlite, + systemtap-lix ? __forDefaults.systemtap-lix, toml11, util-linuxMinimal ? utillinuxMinimal, utillinuxMinimal ? null, @@ -77,6 +80,11 @@ lintInsteadOfBuild ? false, + # FIXME(jade): figure out if it is possible to support non-linux systems for dtrace probes + withDtrace ? + lib.meta.availableOn stdenv.hostPlatform libsystemtap + && lib.meta.availableOn stdenv.buildPlatform systemtap-lix, + # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults. __forDefaults ? { @@ -99,6 +107,10 @@ propagatedBuildInputs = (prev.propagatedBuildInputs or [ ]) ++ [ ncurses ]; }); + # Avoid a bunch of build closure of the tracer, we just need the dtrace + # generator. + systemtap-lix = buildPackages.linuxPackages.systemtap.override { withStap = false; }; + build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; # needs derivation patching to add debuginfo and coroutine library support @@ -117,6 +129,13 @@ let version = __forDefaults.versionJson.version + versionSuffix; + # This could be the dtrace for macOS, etc, but I have no idea if it is + # packaged or if it works. + dtrace-generator = lib.optional withDtrace systemtap-lix; + + # This is for sys/sdt.h + dtrace-headers = lib.optional withDtrace libsystemtap; + aws-sdk-cpp-nix = if aws-sdk-cpp == null then null @@ -238,6 +257,7 @@ stdenv.mkDerivation (finalAttrs: { # dependencies for. (lib.mesonEnable "gc" enableGC) (lib.mesonEnable "internal-api-docs" internalApiDocs) + (lib.mesonEnable "dtrace-probes" withDtrace) (lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild)) (lib.mesonBool "enable-docs" canRunInstalled) (lib.mesonBool "werror" werror) @@ -258,6 +278,7 @@ stdenv.mkDerivation (finalAttrs: { capnproto-lix # Required for libstd++ assertions that leaks inside of the final binary. removeReferencesTo + dtrace-generator ] ++ [ (lib.getBin lowdown-unsandboxed) @@ -304,6 +325,7 @@ stdenv.mkDerivation (finalAttrs: { toml11 pegtl capnproto-lix + dtrace-headers ] ++ lib.optionals hostPlatform.isLinux [ libseccomp