From 8a5a477ca32a9c90c41fbdd16d6f320f1e5d2542 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 14 Jul 2025 17:17:12 +0200 Subject: [PATCH] treewide: add first batch of capnp rpc types this touches both libutil and libstore because with no rpc users it doesn't make that much sense to separate the two. note that all our strings are represented as Data (ie, blobs) because capnp Text must be nul-terminated. while it's technically possible to use Text with strings containing non-terminating NULs it is a bit of a hassle and could lead to rpc users erroneously stopping at the first NUL byte. Change-Id: I4c75e03b79a226ffa8d7cd985e3ac632a0cd7c1c --- lix/libstore/meson.build | 38 ++++++++ lix/libstore/types-rpc.hh | 26 +++++ lix/libstore/types.capnp | 8 ++ lix/libutil/meson.build | 43 +++++++++ lix/libutil/rpc-fwd.hh | 10 ++ lix/libutil/rpc.hh | 195 ++++++++++++++++++++++++++++++++++++++ lix/libutil/types-rpc.hh | 130 +++++++++++++++++++++++++ lix/libutil/types.capnp | 36 +++++++ 8 files changed, 486 insertions(+) create mode 100644 lix/libstore/types-rpc.hh create mode 100644 lix/libstore/types.capnp create mode 100644 lix/libutil/rpc-fwd.hh create mode 100644 lix/libutil/rpc.hh create mode 100644 lix/libutil/types-rpc.hh create mode 100644 lix/libutil/types.capnp diff --git a/lix/libstore/meson.build b/lix/libstore/meson.build index 5205539b4..a9d117251 100644 --- a/lix/libstore/meson.build +++ b/lix/libstore/meson.build @@ -1,4 +1,5 @@ libstore_generated_headers = [] +libstore_generated_sources = [] foreach header : [ 'schema.sql' ] libstore_generated_headers += custom_target( command : [ 'bash', '-c', 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\'' ], @@ -10,6 +11,40 @@ foreach header : [ 'schema.sql' ] ) endforeach +libstore_rpc = [] + +libstore_rpc += custom_target( + command : [ + capnpc_wrapper, + '--language=c++', + '--src-prefix=@CURRENT_SOURCE_DIR@', + '--outdir=@OUTDIR@', + '--depfile=@DEPFILE@', + '-I@SOURCE_ROOT@', + '@INPUT@', + ], + input : files( + # keep-sorted start + 'types.capnp', + # keep-sorted end + ), + output : [ + '@PLAINNAME@.h', + '@PLAINNAME@.c++', + ], + depfile : '@PLAINNAME@.d', +) + +foreach i : libstore_rpc + foreach rpc_src : i.to_list() + if rpc_src.full_path().endswith('.h') + libstore_generated_headers += [ rpc_src ] + else + libstore_generated_sources += [ rpc_src ] + endif + endforeach +endforeach + libstore_settings_headers = [] file_transfer_setting_definitions = files( @@ -268,6 +303,7 @@ libstore_headers = files( 'store-api.hh', 'store-cast.hh', 'temporary-dir.hh', + 'types-rpc.hh', 'uds-remote-store.hh', 'worker-protocol-impl.hh', 'worker-protocol.hh', @@ -356,6 +392,7 @@ dependencies = [ aws_sdk_transfer, nlohmann_json, kj, + capnp_rpc, ] if host_machine.system() == 'freebsd' @@ -365,6 +402,7 @@ endif libstore = library( 'lixstore', libstore_sources, + libstore_generated_sources, libstore_settings_headers, libstore_generated_headers, libstore_extra_objects, diff --git a/lix/libstore/types-rpc.hh b/lix/libstore/types-rpc.hh new file mode 100644 index 000000000..426ed8acb --- /dev/null +++ b/lix/libstore/types-rpc.hh @@ -0,0 +1,26 @@ +#pragma once +///@file RPC helper functions for `types.hh` + +#include "lix/libstore/types.capnp.h" +#include "lix/libutil/rpc.hh" +#include "path.hh" +#include "store-api.hh" +#include + +namespace nix::rpc { +inline nix::StorePath from(const StorePath::Reader & sp, auto &&... args) +{ + Store & store = std::get(std::tie(args...)); + return store.parseStorePath(to(sp.getRaw())); +} + +template<> +struct Fill +{ + static void fill(StorePath::Builder spb, const nix::StorePath & sp, auto &&... args) + { + Store & store = std::get(std::tie(args...)); + LIX_RPC_FILL(spb, setRaw, store.printStorePath(sp)); + } +}; +} diff --git a/lix/libstore/types.capnp b/lix/libstore/types.capnp new file mode 100644 index 000000000..7caf9f563 --- /dev/null +++ b/lix/libstore/types.capnp @@ -0,0 +1,8 @@ +@0x8f8131bf93ce599c; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("nix::rpc"); + +struct StorePath { + raw @0 :Data; +} diff --git a/lix/libutil/meson.build b/lix/libutil/meson.build index 4881e56dd..19dba69f4 100644 --- a/lix/libutil/meson.build +++ b/lix/libutil/meson.build @@ -116,6 +116,8 @@ libutil_headers = files( 'regex.hh', 'repair-flag.hh', 'result.hh', + 'rpc-fwd.hh', + 'rpc.hh', 'serialise.hh', 'shlex.hh', 'signals.hh', @@ -130,6 +132,7 @@ libutil_headers = files( 'thread-pool.hh', 'topo-sort.hh', 'tracepoint.hh', + 'types-rpc.hh', 'types.hh', 'unix-domain-socket.hh', 'url-name.hh', @@ -287,9 +290,44 @@ libutil_settings_headers += custom_target( install_dir : includedir / 'lix/libutil', ) +libutil_rpc_headers = [] +libutil_rpc_sources = [] +libutil_rpc = [] + +libutil_rpc += custom_target( + command : [ + capnpc_wrapper, + '--language=c++', + '--src-prefix=@CURRENT_SOURCE_DIR@', + '--outdir=@OUTDIR@', + '--depfile=@DEPFILE@', + '-I@SOURCE_ROOT@', + '@INPUT@', + ], + input : files( + # keep-sorted start + 'types.capnp', + # keep-sorted end + ), + output : [ + '@PLAINNAME@.h', + '@PLAINNAME@.c++', + ], + install : true, + install_dir : [includedir / 'lix/libutil', false], + depfile : '@PLAINNAME@.d', +) + +foreach rpc : libutil_rpc + libutil_rpc_headers += rpc[0] + libutil_rpc_sources += rpc[1] +endforeach + libutil = library( 'lixutil', libutil_sources, + libutil_rpc_sources, + libutil_rpc_headers, experimental_features_header, experimental_features_impl_header, deprecated_features_header, @@ -307,6 +345,7 @@ libutil = library( openssl, nlohmann_json, kj, + capnp_rpc, ], include_directories : [ '../..' ], cpp_pch : cpp_pch, @@ -337,6 +376,7 @@ liblixutil = declare_dependency( experimental_features_header, deprecated_features_header, libutil_settings_headers, + libutil_rpc_headers, ], dependencies: [ boost, @@ -344,6 +384,7 @@ liblixutil = declare_dependency( # lix-base pkg-config externally) kj, libarchive, + capnp_rpc, ], link_with : libutil ) @@ -356,6 +397,7 @@ if is_static experimental_features_header, deprecated_features_header, libutil_settings_headers, + libutil_rpc_headers, ], dependencies: [ boost, @@ -363,6 +405,7 @@ if is_static # lix-base pkg-config externally) kj, libarchive, + capnp_rpc, ], link_whole : libutil, ) diff --git a/lix/libutil/rpc-fwd.hh b/lix/libutil/rpc-fwd.hh new file mode 100644 index 000000000..2534be5fd --- /dev/null +++ b/lix/libutil/rpc-fwd.hh @@ -0,0 +1,10 @@ +#pragma once +///@file RPC helper forward declarations + +namespace nix::rpc { +template +struct Convert; + +template +struct Fill; +} diff --git a/lix/libutil/rpc.hh b/lix/libutil/rpc.hh new file mode 100644 index 000000000..9aa36684c --- /dev/null +++ b/lix/libutil/rpc.hh @@ -0,0 +1,195 @@ +#pragma once +///@file RPC helper functions + +#include "lix/libutil/charptr-cast.hh" +#include "lix/libutil/rpc-fwd.hh" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix::rpc { + +template + requires requires(From f) { Convert{}; } +To to(const From & from, auto &&... args) +{ + return Convert::convert(from, args...); +} + +// conversions for capnp Text +template<> +struct Convert +{ + static std::string_view convert(const capnp::Text::Reader & t, auto &&...) + { + return std::string_view(t.begin(), t.size()); + } +}; + +template<> +struct Convert +{ + static std::string convert(const capnp::Text::Reader & t, auto &&...) + { + return std::string(to(t)); + } +}; + +// conversions for capnp Data +template<> +struct Convert +{ + static std::string_view convert(const capnp::Data::Reader & t, auto &&...) + { + return std::string_view(t.asChars().begin(), t.size()); + } +}; + +template<> +struct Convert +{ + static std::string convert(const capnp::Data::Reader & t, auto &&...) + { + return std::string(to(t)); + } +}; + +// conversions for collections. order is maintained during conversion and turns +// into insertion order for the result, i.e. sets and maps will only retain the +// first element in the list that compares equivalent to some other list entry. +template + requires requires(To t, capnp::List::Reader list) { + typename To::value_type; + t.insert(t.end(), to(list[0])); + } +struct Convert, To> +{ + static To convert(const capnp::List::Reader & list, auto &&... args) + { + To result; + for (auto && t : list) { + result.insert(result.end(), to(t, args...)); + } + return result; + } +}; + +template + requires std::convertible_to, std::string> +struct Fill, From> +{ + static void fill(capnp::List::Builder builder, const From & from, auto &&... args) + { + size_t i = 0; + for (const std::string & e : from) { + builder.set(i++, e); + } + } +}; + +template + requires std::convertible_to, std::string_view> +struct Fill, From> +{ + static void fill(capnp::List::Builder builder, const From & from, auto &&... args) + { + size_t i = 0; + for (auto && e : from) { + std::string_view sv = e; + builder.set(i++, {charptr_cast(sv.begin()), sv.size()}); + } + } +}; + +template + requires requires { Fill>{}; } +struct Fill, From> +{ + static void fill(capnp::List::Builder builder, const From & from, auto &&... args) + { + size_t i = 0; + for (auto && e : from) { + Fill>::fill(builder[i++], e, args...); + } + } +}; + +// blanket converter that turns a `from` into a `to` +template + requires requires(typename Rpc::Reader r) { + { + from(r) + } -> std::same_as; + } +struct Convert +{ + static To convert(const typename Rpc::Reader & r, auto &&... args) + { + return from(r, args...); + } +}; + +namespace detail { +// strings to Data and Text +template From> +inline void +doFill(auto && builder, void (Builder::*Field)(::capnp::Data::Reader), From && f, auto &&...) +{ + std::string_view sv = f; + (builder.*Field)({charptr_cast(sv.begin()), sv.size()}); +} + +template From> +inline void +doFill(auto && builder, void (Builder::*field)(::capnp::Text::Reader), From && f, auto &&...) +{ + (builder.*field)(f); +} + +// structs +template +inline void doFill(auto && builder, Inner (Builder::*field)(), From && f, auto &&... args) +{ + Fill>::fill( + (builder.*field)(), std::forward(f), args... + ); +} + +// lists and other primitives. lists must be distinguished by having builders. +template +inline void doFill(auto && builder, Inner (Builder::*field)(Init), From && f, auto &&... args) +{ + if constexpr (requires { + std::same_as; + typename Inner::Builds; + }) + { + Fill>::fill( + (builder.*field)(f.size()), std::forward(f), args... + ); + } else { + (builder.*field)(std::forward(f)); + } +} +} + +#define LIX_RPC_FILL(fobj, ffield, fsource, ...) \ + [&] { ::nix::rpc::detail::doFill(fobj, &decltype(fobj)::ffield, (fsource), ##__VA_ARGS__); }() + +#define LIX_TRY_AWAIT_RPC(...) \ + LIX_TRY_AWAIT_CONTEXT_MAP( \ + [] { return "RPC call"; }, \ + ([](auto r) { return ::nix::rpc::from(r.getResult()); }), \ + __VA_ARGS__ \ + ) + +#ifdef LIX_UR_COMPILER_UWU +#define RPC_FILL LIX_RPC_FILL +#define TRY_AWAIT_RPC LIX_TRY_AWAIT_RPC +#endif +} diff --git a/lix/libutil/types-rpc.hh b/lix/libutil/types-rpc.hh new file mode 100644 index 000000000..b5e331a55 --- /dev/null +++ b/lix/libutil/types-rpc.hh @@ -0,0 +1,130 @@ +#pragma once +///@file RPC helper functions for `types.hh` + +#include "error.hh" +#include "lix/libutil/result.hh" +#include "lix/libutil/types.capnp.h" +#include "rpc.hh" +#include +#include +#include +#include +#include +#include + +namespace nix::rpc { + +// ensure that verbosity levels match to make conversions trivial below. +static_assert(int(Verbosity::ERROR) == int(nix::Verbosity::lvlError)); +static_assert(int(Verbosity::WARN) == int(nix::Verbosity::lvlWarn)); +static_assert(int(Verbosity::NOTICE) == int(nix::Verbosity::lvlNotice)); +static_assert(int(Verbosity::INFO) == int(nix::Verbosity::lvlInfo)); +static_assert(int(Verbosity::TALKATIVE) == int(nix::Verbosity::lvlTalkative)); +static_assert(int(Verbosity::CHATTY) == int(nix::Verbosity::lvlChatty)); +static_assert(int(Verbosity::DEBUG) == int(nix::Verbosity::lvlDebug)); +static_assert(int(Verbosity::VOMIT) == int(nix::Verbosity::lvlVomit)); + +inline nix::ErrorInfo from(const Error::Reader & e, auto &&... args) +{ + ErrorInfo ei{ + // capnp enums are u16, and we have checked that the enumerator values match + .level = nix::Verbosity(std::clamp>( + uint16_t(e.getLevel()), lvlError, lvlVomit + )), + .msg = HintFmt(to(e.getMessage())), + }; + for (auto && t : e.getTraces()) { + ei.traces.push_back(Trace{.hint = HintFmt(to(t, args...))}); + } + return ei; +} + +template<> +struct Fill +{ + static void fill(Error::Builder eb, const nix::ErrorInfo & e, auto &&...) + { + eb.setLevel(Verbosity(e.level)); + LIX_RPC_FILL(eb, setMessage, e.msg.str()); + LIX_RPC_FILL(eb, initTraces, e.traces | std::ranges::views::transform([](auto & trace) { + return trace.hint.str(); + })); + } +}; + +namespace detail { +inline void makeBadResult(auto rb, const std::exception_ptr & e) +{ + try { + std::rethrow_exception(e); + } catch (nix::Error & e) { + LIX_RPC_FILL(rb, initBad, e.info()); + } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) + LIX_RPC_FILL(rb, initBad, nix::Error("caught non-lix exception: %s", e.what()).info()); + } catch (...) { + LIX_RPC_FILL(rb, initBad, nix::Error("caught non-exception! spooky").info()); + } +} + +template +inline constexpr bool IsResult = false; +template +inline constexpr bool IsResult> = true; + +template +concept ResultReader = IsResult; + +template +concept ResultBuilder = IsResult; +} + +inline nix::Result from(const ResultV::Reader & r, auto &&... args) +{ + if (r.isGood()) { + return result::success(); + } else { + return result::failure(nix::Error(from(r.getBad(), args...))); + } +} + +template<> +struct Fill> +{ + static void fill(ResultV::Builder rb, const nix::Result & r, auto &&...) + { + if (r.has_value()) { + rb.setGood(); + } else { + detail::makeBadResult(rb, r.error()); + } + } +}; + +template<> +struct Fill +{ + static void fill(ResultV::Builder rb, const std::exception_ptr & e, auto &&...) + { + detail::makeBadResult(rb, e); + } +}; + +inline auto from(detail::ResultReader auto r, auto &&... args) +{ + using R = nix::Result; + if (r.isGood()) { + return R(result::success(r.getGood())); + } else { + return R(result::failure(nix::Error(from(r.getBad(), args...)))); + } +} + +template +struct Fill, std::exception_ptr> +{ + static void fill(Result::Builder rb, const std::exception_ptr & e, auto &&...) + { + detail::makeBadResult(rb, e); + } +}; +} diff --git a/lix/libutil/types.capnp b/lix/libutil/types.capnp new file mode 100644 index 000000000..0b25d5eef --- /dev/null +++ b/lix/libutil/types.capnp @@ -0,0 +1,36 @@ +@0xd43cca581e0ebf82; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("nix::rpc"); + +enum Verbosity { + error @0; + warn @1; + notice @2; + info @3; + talkative @4; + chatty @5; + debug @6; + vomit @7; +} + +struct Error { + level @0 :Verbosity; + message @1 :Data; + traces @2 :List(Data); +} + +struct Result(T) { + union { + good @0 :T; + bad @1 :Error; + } +} + +# primitives can't be args to generics, so we need to specialize for primitive here. +struct ResultV { + union { + good @0 :Void; + bad @1 :Error; + } +}