diff --git a/doc/manual/rl-next/usdt-support.md b/doc/manual/rl-next/usdt-support.md
new file mode 100644
index 000000000..e741f7911
--- /dev/null
+++ b/doc/manual/rl-next/usdt-support.md
@@ -0,0 +1,35 @@
+---
+synopsis: "Add support for eBPF USDT/dtrace probes inside Lix"
+issues: [fj#727]
+cls: [2884]
+category: Features
+credits: jade
+---
+
+eBPF tracers like `bpftrace` and `dtrace` are a group of similar tools for debugging production systems.
+User-space statically defined tracing probes (USDT) allow for defining zero or near-zero disabled-probe-effect probes, thus allowing instrumentation of hot paths in production builds.
+Lix now has internal support for defining these probes and has shipped its first probe.
+
+As of this writing it is available by default in the Linux build of Lix.
+
+To try it out on Linux, you can use the following example command:
+
+```
+$ sudo bpftrace -l 'usdt:/path/to/liblixstore.so:*:*'
+usdt:/path/to/liblixstore.so:lix_store:filetransfer__read
+
+$ sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }'
+Attaching 1 probe...
+https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192
+https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192
+https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536
+https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536
+```
+
+Note that bpftrace does not offer any way to list the arguments to USDT probes in a human readable form.
+To get the probe definitions, see the `*.d` files in the Lix source code, for example, `lix/libstore/trace-probes.d`.
+
+For more resources on eBPF/bpftrace and dtrace, see:
+* The book "BPF Performance Tools" by Brendan Gregg, which discusses bpftrace at length.
+*
+* [Illumos' dtrace book](https://illumos.org/books/dtrace/preface.html)
diff --git a/lix/libstore/filetransfer.cc b/lix/libstore/filetransfer.cc
index f94691974..cfc340874 100644
--- a/lix/libstore/filetransfer.cc
+++ b/lix/libstore/filetransfer.cc
@@ -8,6 +8,7 @@
#include "lix/libutil/signals.hh"
#include "lix/libutil/strings.hh"
#include "lix/libutil/thread-name.hh"
+#include "lix/libutil/tracepoint.hh"
#include
#include
@@ -947,6 +948,8 @@ struct curlFileTransfer : public FileTransfer
size_t read(char * data, size_t len) override
{
+ TRACE(DTRACE_PROBE2(lix_store, filetransfer__read, uri.c_str(), len));
+
size_t total = 0;
while (total < len && awaitData()) {
const auto available = std::min(len - total, buffered.size());
diff --git a/lix/libstore/meson.build b/lix/libstore/meson.build
index 2b2616d06..27145df8d 100644
--- a/lix/libstore/meson.build
+++ b/lix/libstore/meson.build
@@ -128,6 +128,12 @@ libstore_settings_headers += custom_target(
install_dir : includedir / 'lix/libstore',
)
+libstore_extra_objects = []
+if dtrace_feature.enabled()
+ libstore_settings_headers += dtrace_header_gen.process('trace-probes.d')
+ libstore_extra_objects += dtrace_object_gen.process('trace-probes.d')
+endif
+
libstore_sources = files(
# keep-sorted start
'binary-cache-store.cc',
@@ -358,6 +364,7 @@ libstore = library(
libstore_sources,
libstore_settings_headers,
libstore_generated_headers,
+ libstore_extra_objects,
include_directories : [ '../..' ],
dependencies : dependencies,
cpp_args : cpp_args,
diff --git a/lix/libstore/trace-probes.d b/lix/libstore/trace-probes.d
new file mode 100644
index 000000000..580b00310
--- /dev/null
+++ b/lix/libstore/trace-probes.d
@@ -0,0 +1,9 @@
+// I don't know how the rest of these stability attrs work and tbh I don't care
+// either; these probes are not stable API and we don't need to be more
+// specific about exactly how.
+#pragma D attributes Unstable/Unstable/Common provider lix_store provider
+
+provider lix_store {
+ /** See filetransfer.cc; this is the consumption side, not the curl/production side. */
+ probe filetransfer__read(string url, size_t length);
+};
diff --git a/lix/libutil/meson.build b/lix/libutil/meson.build
index a78f67e5b..642b87bcc 100644
--- a/lix/libutil/meson.build
+++ b/lix/libutil/meson.build
@@ -129,6 +129,7 @@ libutil_headers = files(
'thread-name.hh',
'thread-pool.hh',
'topo-sort.hh',
+ 'tracepoint.hh',
'types.hh',
'unix-domain-socket.hh',
'url-name.hh',
diff --git a/lix/libutil/tracepoint.hh b/lix/libutil/tracepoint.hh
new file mode 100644
index 000000000..edbbc0c4b
--- /dev/null
+++ b/lix/libutil/tracepoint.hh
@@ -0,0 +1,24 @@
+#pragma once
+/** @file USDT based trace points
+ *
+ * These can be used with bpftrace or dtrace using their respective USDT trace
+ * providers.
+ *
+ * See the .d files in each library for probe details.
+ *
+ * Example:
+ *
+ * ```
+ * sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }'
+ * ```
+ */
+
+#include "lix/config.h"
+
+// NOTE: glib disables this for the clang static analyzer, idk if we need to also
+#if HAVE_DTRACE
+#define TRACE(body) body
+#include
+#else
+#define TRACE(body)
+#endif
diff --git a/meson.build b/meson.build
index d66ff1a8c..8a23590e1 100644
--- a/meson.build
+++ b/meson.build
@@ -402,6 +402,32 @@ endif
# FIXME(Qyriad): the autoconf system checks that busybox has the "standalone" feature, indicating
# that busybox sh won't run busybox applets as builtins (which would break our sandbox).
+# Trace points for dtrace and bpftrace
+dtrace_feature = get_option('dtrace-probes')
+if dtrace_feature.allowed()
+ dtrace_exe = find_program('dtrace', native : false, required : false)
+ dtrace_header_present = cxx.has_header('sys/sdt.h')
+ dtrace_feature = dtrace_feature.enable_auto_if(dtrace_exe.found() and dtrace_header_present)
+endif
+
+if dtrace_feature.enabled()
+ dtrace_feature.require(dtrace_exe.found() and dtrace_header_present,
+ error_message : 'trace probes require both the dtrace command and sys/sdt.h header to be available')
+
+ # NOTE: glib seems to have had to hack some stuff up that we are not hacking
+ # up. I don't know why.
+ # https://github.com/GNOME/glib/blob/03f7c1fbf3a3784cb4c3604f83ca3645e9225577/meson.build#L2420-L2434
+ dtrace_object_gen = generator(dtrace_exe,
+ output : '@BASENAME@.o',
+ arguments : ['-G', '-s', '@INPUT@', '-o', '@OUTPUT@'])
+ dtrace_header_gen = generator(dtrace_exe,
+ output : '@BASENAME@.gen.h',
+ arguments : ['-h', '-s', '@INPUT@', '-o', '@OUTPUT@'])
+endif
+configdata += {
+ 'HAVE_DTRACE': dtrace_feature.enabled().to_int(),
+}
+
lsof = find_program('lsof', native : true)
# This is how Nix does generated headers...
diff --git a/meson.options b/meson.options
index 81efba8b9..8d5eed0bc 100644
--- a/meson.options
+++ b/meson.options
@@ -73,6 +73,10 @@ option('enable-pch-std', type : 'boolean', value : true,
description : 'whether to use precompiled headers for C++\'s standard library (breaks clangd if you\'re using GCC)',
)
+option('dtrace-probes', type : 'feature', value : 'auto',
+ description : 'whether to build dtrace/bpftrace compatible USDT probes into Lix'
+)
+
option('lix-clang-tidy-checks-path', type : 'string', value : '',
description: 'path to lix-clang-tidy-checks library file, if providing it externally. Uses an internal one if this is not set',
)
diff --git a/package.nix b/package.nix
index 31fc68145..5a2f0d166 100644
--- a/package.nix
+++ b/package.nix
@@ -30,6 +30,8 @@
libcpuid,
libseccomp,
libsodium,
+ libsystemtap,
+ linuxPackages,
lix-clang-tidy ? null,
llvmPackages,
lsof,
@@ -51,6 +53,7 @@
rustPlatform,
rustc,
sqlite,
+ systemtap-lix ? __forDefaults.systemtap-lix,
toml11,
util-linuxMinimal ? utillinuxMinimal,
utillinuxMinimal ? null,
@@ -77,6 +80,11 @@
lintInsteadOfBuild ? false,
+ # FIXME(jade): figure out if it is possible to support non-linux systems for dtrace probes
+ withDtrace ?
+ lib.meta.availableOn stdenv.hostPlatform libsystemtap
+ && lib.meta.availableOn stdenv.buildPlatform systemtap-lix,
+
# Not a real argument, just the only way to approximate let-binding some
# stuff for argument defaults.
__forDefaults ? {
@@ -99,6 +107,10 @@
propagatedBuildInputs = (prev.propagatedBuildInputs or [ ]) ++ [ ncurses ];
});
+ # Avoid a bunch of build closure of the tracer, we just need the dtrace
+ # generator.
+ systemtap-lix = buildPackages.linuxPackages.systemtap.override { withStap = false; };
+
build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
# needs derivation patching to add debuginfo and coroutine library support
@@ -117,6 +129,13 @@ let
version = __forDefaults.versionJson.version + versionSuffix;
+ # This could be the dtrace for macOS, etc, but I have no idea if it is
+ # packaged or if it works.
+ dtrace-generator = lib.optional withDtrace systemtap-lix;
+
+ # This is for sys/sdt.h
+ dtrace-headers = lib.optional withDtrace libsystemtap;
+
aws-sdk-cpp-nix =
if aws-sdk-cpp == null then
null
@@ -238,6 +257,7 @@ stdenv.mkDerivation (finalAttrs: {
# dependencies for.
(lib.mesonEnable "gc" enableGC)
(lib.mesonEnable "internal-api-docs" internalApiDocs)
+ (lib.mesonEnable "dtrace-probes" withDtrace)
(lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild))
(lib.mesonBool "enable-docs" canRunInstalled)
(lib.mesonBool "werror" werror)
@@ -258,6 +278,7 @@ stdenv.mkDerivation (finalAttrs: {
capnproto-lix
# Required for libstd++ assertions that leaks inside of the final binary.
removeReferencesTo
+ dtrace-generator
]
++ [
(lib.getBin lowdown-unsandboxed)
@@ -304,6 +325,7 @@ stdenv.mkDerivation (finalAttrs: {
toml11
pegtl
capnproto-lix
+ dtrace-headers
]
++ lib.optionals hostPlatform.isLinux [
libseccomp