observability: add the first USDT probe

USDT probes are statically defined trace points that have nearly zero
disabled-probe effect, i.e. we can put them in hot paths.

The use case for these is both similar and dissimilar to Rust tracing:
We still need better logging and a better structured rust-tracing
looking thing, but probes allow for quite easy programmable interactive
tracing in production, which we also care a lot about.

This CL comes with a perfunctory trace point in
libstore/file-transfer.cc for reading data out of the curl buffer. This
was mostly thrown in there so that I could see what the buffer sizes of
this were, and maybe be able to instrument the perf of the curl usages
in Lix in the future.

Fixes: https://git.lix.systems/lix-project/lix/issues/727
Change-Id: I0f5d9912d76bf3d6923bf53ebfd9b8d6c6e70aea

doc/manual/rl-next/usdt-support.md

@@ -0,0 +1,35 @@
+---
+synopsis: "Add support for eBPF USDT/dtrace probes inside Lix"
+issues: [fj#727]
+cls: [2884]
+category: Features
+credits: jade
+---
+
+eBPF tracers like `bpftrace` and `dtrace` are a group of similar tools for debugging production systems.
+User-space statically defined tracing probes (USDT) allow for defining zero or near-zero disabled-probe-effect probes, thus allowing instrumentation of hot paths in production builds.
+Lix now has internal support for defining these probes and has shipped its first probe.
+
+As of this writing it is available by default in the Linux build of Lix.
+
+To try it out on Linux, you can use the following example command:
+
+```
+$ sudo bpftrace -l 'usdt:/path/to/liblixstore.so:*:*'
+usdt:/path/to/liblixstore.so:lix_store:filetransfer__read
+
+$ sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }'
+Attaching 1 probe...
+https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192
+https://cache.nixos.org/wvpzaycmvs39h5bcsfrxkjsg48mj4h73.narinf.. read 8192
+https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536
+https://cache.nixos.org/nar/1qshsc30nlarzdig0v9b1aasdkwaxhnv0a0.. read 65536
+```
+
+Note that bpftrace does not offer any way to list the arguments to USDT probes in a human readable form.
+To get the probe definitions, see the `*.d` files in the Lix source code, for example, `lix/libstore/trace-probes.d`.
+
+For more resources on eBPF/bpftrace and dtrace, see:
+* The book "BPF Performance Tools" by Brendan Gregg, which discusses bpftrace at length.
+* <https://ebpf.io/get-started/>
+* [Illumos' dtrace book](https://illumos.org/books/dtrace/preface.html)

lix/libstore/filetransfer.cc

@@ -8,6 +8,7 @@
 #include "lix/libutil/signals.hh"
 #include "lix/libutil/strings.hh"
 #include "lix/libutil/thread-name.hh"
+#include "lix/libutil/tracepoint.hh"
 #include <cstddef>
 
 #include <cstdio>
@@ -947,6 +948,8 @@ struct curlFileTransfer : public FileTransfer
 
         size_t read(char * data, size_t len) override
         {
+            TRACE(DTRACE_PROBE2(lix_store, filetransfer__read, uri.c_str(), len));
+
             size_t total = 0;
             while (total < len && awaitData()) {
                 const auto available = std::min(len - total, buffered.size());

lix/libstore/meson.build

@@ -128,6 +128,12 @@ libstore_settings_headers += custom_target(
   install_dir : includedir / 'lix/libstore',
 )
 
+libstore_extra_objects = []
+if dtrace_feature.enabled()
+  libstore_settings_headers += dtrace_header_gen.process('trace-probes.d')
+  libstore_extra_objects += dtrace_object_gen.process('trace-probes.d')
+endif
+
 libstore_sources = files(
   # keep-sorted start
   'binary-cache-store.cc',
@@ -358,6 +364,7 @@ libstore = library(
   libstore_sources,
   libstore_settings_headers,
   libstore_generated_headers,
+  libstore_extra_objects,
   include_directories : [ '../..' ],
   dependencies : dependencies,
   cpp_args : cpp_args,

lix/libstore/trace-probes.d

@@ -0,0 +1,9 @@
+// I don't know how the rest of these stability attrs work and tbh I don't care
+// either; these probes are not stable API and we don't need to be more
+// specific about exactly how.
+#pragma D attributes Unstable/Unstable/Common provider lix_store provider
+
+provider lix_store {
+    /** See filetransfer.cc; this is the consumption side, not the curl/production side. */
+    probe filetransfer__read(string url, size_t length);
+};

lix/libutil/meson.build

@@ -129,6 +129,7 @@ libutil_headers = files(
   'thread-name.hh',
   'thread-pool.hh',
   'topo-sort.hh',
+  'tracepoint.hh',
   'types.hh',
   'unix-domain-socket.hh',
   'url-name.hh',

lix/libutil/tracepoint.hh

@@ -0,0 +1,24 @@
+#pragma once
+/** @file USDT based trace points
+ *
+ * These can be used with bpftrace or dtrace using their respective USDT trace
+ * providers.
+ *
+ * See the .d files in each library for probe details.
+ *
+ * Example:
+ *
+ * ```
+ * sudo bpftrace -e 'usdt:*:lix_store:filetransfer__read { printf("%s read %d\n", str(arg0), arg1); }'
+ * ```
+ */
+
+#include "lix/config.h"
+
+// NOTE: glib disables this for the clang static analyzer, idk if we need to also
+#if HAVE_DTRACE
+#define TRACE(body) body
+#include <sys/sdt.h>
+#else
+#define TRACE(body)
+#endif

meson.build

@@ -402,6 +402,32 @@ endif
 # FIXME(Qyriad): the autoconf system checks that busybox has the "standalone" feature, indicating
 # that busybox sh won't run busybox applets as builtins (which would break our sandbox).
 
+# Trace points for dtrace and bpftrace
+dtrace_feature = get_option('dtrace-probes')
+if dtrace_feature.allowed()
+  dtrace_exe = find_program('dtrace', native : false, required : false)
+  dtrace_header_present = cxx.has_header('sys/sdt.h')
+  dtrace_feature = dtrace_feature.enable_auto_if(dtrace_exe.found() and dtrace_header_present)
+endif
+
+if dtrace_feature.enabled()
+  dtrace_feature.require(dtrace_exe.found() and dtrace_header_present,
+    error_message : 'trace probes require both the dtrace command and sys/sdt.h header to be available')
+
+  # NOTE: glib seems to have had to hack some stuff up that we are not hacking
+  # up. I don't know why.
+  # https://github.com/GNOME/glib/blob/03f7c1fbf3a3784cb4c3604f83ca3645e9225577/meson.build#L2420-L2434
+  dtrace_object_gen = generator(dtrace_exe,
+    output : '@BASENAME@.o',
+    arguments : ['-G', '-s', '@INPUT@', '-o', '@OUTPUT@'])
+  dtrace_header_gen = generator(dtrace_exe,
+    output : '@BASENAME@.gen.h',
+    arguments : ['-h', '-s', '@INPUT@', '-o', '@OUTPUT@'])
+endif
+configdata += {
+  'HAVE_DTRACE': dtrace_feature.enabled().to_int(),
+}
+
 lsof = find_program('lsof', native : true)
 
 # This is how Nix does generated headers...

meson.options

@@ -73,6 +73,10 @@ option('enable-pch-std', type : 'boolean', value : true,
   description : 'whether to use precompiled headers for C++\'s standard library (breaks clangd if you\'re using GCC)',
 )
 
+option('dtrace-probes', type : 'feature', value : 'auto',
+  description : 'whether to build dtrace/bpftrace compatible USDT probes into Lix'
+)
+
 option('lix-clang-tidy-checks-path', type : 'string', value : '',
   description: 'path to lix-clang-tidy-checks library file, if providing it externally. Uses an internal one if this is not set',
 )

package.nix

@@ -30,6 +30,8 @@
   libcpuid,
   libseccomp,
   libsodium,
+  libsystemtap,
+  linuxPackages,
   lix-clang-tidy ? null,
   llvmPackages,
   lsof,
@@ -51,6 +53,7 @@
   rustPlatform,
   rustc,
   sqlite,
+  systemtap-lix ? __forDefaults.systemtap-lix,
   toml11,
   util-linuxMinimal ? utillinuxMinimal,
   utillinuxMinimal ? null,
@@ -77,6 +80,11 @@
 
   lintInsteadOfBuild ? false,
 
+  # FIXME(jade): figure out if it is possible to support non-linux systems for dtrace probes
+  withDtrace ?
+    lib.meta.availableOn stdenv.hostPlatform libsystemtap
+    && lib.meta.availableOn stdenv.buildPlatform systemtap-lix,
+
   # Not a real argument, just the only way to approximate let-binding some
   # stuff for argument defaults.
   __forDefaults ? {
@@ -99,6 +107,10 @@
       propagatedBuildInputs = (prev.propagatedBuildInputs or [ ]) ++ [ ncurses ];
     });
 
+    # Avoid a bunch of build closure of the tracer, we just need the dtrace
+    # generator.
+    systemtap-lix = buildPackages.linuxPackages.systemtap.override { withStap = false; };
+
     build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
 
     # needs derivation patching to add debuginfo and coroutine library support
@@ -117,6 +129,13 @@ let
 
   version = __forDefaults.versionJson.version + versionSuffix;
 
+  # This could be the dtrace for macOS, etc, but I have no idea if it is
+  # packaged or if it works.
+  dtrace-generator = lib.optional withDtrace systemtap-lix;
+
+  # This is for sys/sdt.h
+  dtrace-headers = lib.optional withDtrace libsystemtap;
+
   aws-sdk-cpp-nix =
     if aws-sdk-cpp == null then
       null
@@ -238,6 +257,7 @@ stdenv.mkDerivation (finalAttrs: {
       # dependencies for.
       (lib.mesonEnable "gc" enableGC)
       (lib.mesonEnable "internal-api-docs" internalApiDocs)
+      (lib.mesonEnable "dtrace-probes" withDtrace)
       (lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild))
       (lib.mesonBool "enable-docs" canRunInstalled)
       (lib.mesonBool "werror" werror)
@@ -258,6 +278,7 @@ stdenv.mkDerivation (finalAttrs: {
       capnproto-lix
       # Required for libstd++ assertions that leaks inside of the final binary.
       removeReferencesTo
+      dtrace-generator
     ]
     ++ [
       (lib.getBin lowdown-unsandboxed)
@@ -304,6 +325,7 @@ stdenv.mkDerivation (finalAttrs: {
       toml11
       pegtl
       capnproto-lix
+      dtrace-headers
     ]
     ++ lib.optionals hostPlatform.isLinux [
       libseccomp