diff --git a/lix/libexpr/attr-set.hh b/lix/libexpr/attr-set.hh index 3b3117bb7..fcad59d7d 100644 --- a/lix/libexpr/attr-set.hh +++ b/lix/libexpr/attr-set.hh @@ -44,7 +44,7 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), * elements allocated after this structure, while the size corresponds to * the number of elements already inserted in this structure. */ -class Bindings +class alignas(Value::TAG_ALIGN) Bindings { public: using Size = uint32_t; diff --git a/lix/libexpr/eval-inline.hh b/lix/libexpr/eval-inline.hh index cc7fcba16..3f134e1f0 100644 --- a/lix/libexpr/eval-inline.hh +++ b/lix/libexpr/eval-inline.hh @@ -11,41 +11,35 @@ namespace nix { inline Value::Value(app_t, EvalMemory & mem, Value & lhs, Value & rhs) - : internalType(tApp) - , _app_pad(0) { - _app = static_cast(mem.allocBytes(sizeof(Value::App) + sizeof(Value *))); - _app->_left = &lhs; - _app->_n = 1; - _app->_args[0] = &rhs; + auto app = static_cast(mem.allocBytes(sizeof(Value::App) + sizeof(Value *))); + app->_left = &lhs; + app->_n = 1; + app->_args[0] = &rhs; + raw = tag(tApp, app); } inline Value::Value(app_t, EvalMemory & mem, Value & lhs, std::span args) - : internalType(tApp) - , _app_pad(0) { - _app = static_cast(mem.allocBytes(sizeof(Value::App) + args.size_bytes())); - _app->_left = &lhs; - _app->_n = args.size(); - memcpy(_app->_args, args.data(), args.size_bytes()); + auto app = static_cast(mem.allocBytes(sizeof(Value::App) + args.size_bytes())); + app->_left = &lhs; + app->_n = args.size(); + memcpy(app->_args, args.data(), args.size_bytes()); + raw = tag(tApp, app); } inline Value::Value(thunk_t, EvalMemory & mem, Env & env, Expr & expr) - : internalType(tThunk) - , _thunk_pad(0) { auto thunk = mem.allocType(); *thunk = {.env = &env, .expr = &expr}; - _thunk = thunk; + raw = tag(tThunk, thunk); } inline Value::Value(lambda_t, EvalMemory & mem, Env & env, ExprLambda & lambda) - : internalType(tAuxiliary) - , _aux_pad(0) { auto lp = mem.allocType(); *lp = Lambda{{Acb::tLambda}, &env, &lambda}; - _auxiliary = lp; + raw = tag(tAuxiliary, lp); } [[gnu::always_inline]] diff --git a/lix/libexpr/eval.cc b/lix/libexpr/eval.cc index 5a4d7eba8..f2bd7a4c8 100644 --- a/lix/libexpr/eval.cc +++ b/lix/libexpr/eval.cc @@ -112,7 +112,7 @@ std::string showType(const Value & v) // Allow selecting a subset of enum values #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" - switch (v.internalType) { + switch (v.internalType()) { case tString: return v.string().context ? "a string with context" : "a string"; case tAuxiliary: @@ -182,7 +182,9 @@ void initLibExpr() /* Don't look for interior pointers. This reduces the odds of misdetection a bit. */ GC_set_all_interior_pointers(0); - GC_REGISTER_DISPLACEMENT(1); + for (int i = 1; i < 8; i++) { + GC_REGISTER_DISPLACEMENT(i); + } /* We don't have any roots in data segments, so don't scan from there. */ diff --git a/lix/libexpr/value.cc b/lix/libexpr/value.cc index 5a7ed0485..d6129dbae 100644 --- a/lix/libexpr/value.cc +++ b/lix/libexpr/value.cc @@ -14,6 +14,12 @@ Value Value::EMPTY_LIST{Value::list_t{}, &emptyListData}; const Value::Null Value::NULL_ACB = {{Value::Acb::tNull}}; +static_assert(alignof(Value::String) >= Value::TAG_ALIGN); +static_assert(alignof(Bindings) >= Value::TAG_ALIGN); +static_assert(alignof(Value::List) >= Value::TAG_ALIGN); +static_assert(alignof(Value::Thunk) >= Value::TAG_ALIGN); +static_assert(alignof(Value::App) >= Value::TAG_ALIGN); + static void copyContextToValue(Value::String & s, const NixStringContext & context) { if (!context.empty()) { @@ -25,10 +31,7 @@ static void copyContextToValue(Value::String & s, const NixStringContext & conte } } -Value::Value(primop_t, PrimOp & primop) : internalType(tAuxiliary), _auxiliary(&primop), _aux_pad(0) -{ -} - +Value::Value(primop_t, PrimOp & primop) : raw(tag(tAuxiliary, &primop)) {} void Value::print(EvalState & state, std::ostream & str, PrintOptions options) { @@ -37,8 +40,8 @@ void Value::print(EvalState & state, std::ostream & str, PrintOptions options) bool Value::isTrivial() const { - return internalType != tApp - && (internalType != tThunk + return internalType() != tApp + && (internalType() != tThunk || (thunk().expr->try_cast() && static_cast(thunk().expr)->dynamicAttrs.empty()) || thunk().expr->try_cast() || thunk().expr->try_cast()); @@ -57,13 +60,13 @@ void Value::mkString(std::string_view s) void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); - copyContextToValue(*const_cast(_string), context); + copyContextToValue(*untag(), context); } void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); - copyContextToValue(*const_cast(_string), context); + copyContextToValue(*untag(), context); } void Value::mkPath(const SourcePath & path) diff --git a/lix/libexpr/value.hh b/lix/libexpr/value.hh index 85d6a1ac8..693f45b28 100644 --- a/lix/libexpr/value.hh +++ b/lix/libexpr/value.hh @@ -4,10 +4,12 @@ #include #include #include +#include #include #include #include #include +#include #include "lix/libexpr/gc-alloc.hh" #include "lix/libexpr/value/context.hh" @@ -70,8 +72,10 @@ struct PrimOpDetails std::optional experimentalFeature; }; +// NOTE value.cc contains alignment assertions for pointers tagged thusly. +// *always* ensure that these assertions match the tag types declared here typedef enum { - tInt = 1, + tInt, tBool, tString, tAttrs, @@ -241,7 +245,43 @@ struct NewValueAs struct Value { private: - InternalType internalType; + uintptr_t raw; + +public: + static constexpr size_t TAG_BITS = 3; + static constexpr size_t TAG_ALIGN = 1 << TAG_BITS; + static constexpr uintptr_t TAG_MASK = (1 << TAG_BITS) - 1; + +private: + // boehmgc always allocate in two-word chunks, which means 8 bytes on 32 bit architectures. + // ensure that malloc must always use at least 8 byte chunks as well so our tags always fit + static_assert(alignof(std::max_align_t) >= Value::TAG_ALIGN); + + static uintptr_t tag(InternalType t, auto v) + { + if constexpr (std::is_null_pointer_v) { + return t; + } else if constexpr (std::is_pointer_v) { + return (reinterpret_cast(v)) | t; + } else { + return (static_cast(v) << TAG_BITS) | t; + } + } + + template + T untag() const + { + if constexpr (std::is_pointer_v) { + return reinterpret_cast(raw & ~TAG_MASK); + } else { + return static_cast((raw & ~TAG_MASK) >> TAG_BITS); + } + } + + InternalType internalType() const + { + return InternalType(raw & TAG_MASK); + } friend std::string showType(const Value & v); @@ -294,41 +334,39 @@ public: /// Default constructor which is still used in the codebase but should not /// be used in new code. Zero initializes its members. - [[deprecated]] Value() - : internalType(static_cast(0)) - , _empty{ 0, 0 } - { } + [[deprecated]] + Value() + : raw{0} + { + } /// Constructs a nix language value of type "int", with the integral value /// of @ref i. Value(integer_t, NixInt i) - : internalType(tInt) - , _empty{ 0, 0 } { if (isTaggableInteger(i)) { - _integer = i; + raw = tInt | (uintptr_t(i.value) << TAG_BITS); } else { - internalType = tAuxiliary; auto ip = gcAllocType(); ip->type = Acb::tInt; ip->value = i; - _auxiliary = ip; + raw = tag(tAuxiliary, ip); } } /// Constructs a nix language value of type "float", with the floating /// point value of @ref f. - Value(floating_t, NixFloat f) : internalType(tAuxiliary), _aux_pad(0) + Value(floating_t, NixFloat f) { auto fp = gcAllocType(); fp->type = Acb::tFloat; fp->value = f; - _auxiliary = fp; + raw = tag(tAuxiliary, fp); } /// Constructs a nix language value of type "bool", with the boolean /// value of @ref b. - Value(boolean_t, bool b) : internalType(tBool), _boolean(b), _bool_pad(0) {} + Value(boolean_t, bool b) : raw(tag(tBool, b)) {} /// Constructs a nix language value of type "string", with the value of the /// C-string pointed to by @ref strPtr, and optionally with an array of @@ -338,15 +376,13 @@ public: /// assumes suitable memory has already been allocated (with the GC if /// enabled), and string and context data copied into that memory. Value(string_t, char const * strPtr, char const ** contextPtr = nullptr) - : internalType(tString) - , _string_pad(0) { auto block = gcAllocType(); *block = {.content = strPtr, .context = contextPtr}; - _string = block; + raw = tag(tString, block); } - Value(string_t, const String * str) : internalType(tString), _string(str), _string_pad(0) {} + Value(string_t, const String * str) : raw(tag(tString, str)) {} /// Constructx a nix language value of type "string", with a copy of the /// string data viewed by @ref copyFrom. @@ -354,12 +390,10 @@ public: /// The string data *is* copied from @ref copyFrom, and this constructor /// performs a dynamic (GC) allocation to do so. Value(string_t, std::string_view copyFrom, NixStringContext const & context = {}) - : internalType(tString) - , _string_pad(0) { auto block = gcAllocType(); *block = {.content = gcCopyStringIfNeeded(copyFrom), .context = nullptr}; - _string = block; + raw = tag(tString, block); if (context.empty()) { // It stays nullptr. @@ -389,12 +423,10 @@ public: /// @ref context, and this constructor performs a dynamic (GC) allocation /// to do so. Value(string_t, char const * strPtr, NixStringContext const & context) - : internalType(tString) - , _string_pad(0) { auto block = gcAllocType(); *block = {.content = strPtr, .context = nullptr}; - _string = block; + raw = tag(tString, block); if (context.empty()) { // It stays nullptr @@ -420,7 +452,7 @@ public: /// The C-string is not copied; this constructor assumes suitable memory /// has already been allocated (with the GC if enabled), and string data /// has been copied into that memory. - Value(path_t, const String * str) : internalType(tString), _string(str), _string_pad(0) + Value(path_t, const String * str) : raw(tag(tString, str)) { assert(str->isPath()); } @@ -430,11 +462,11 @@ public: /// /// The data from @ref path *is* copied, and this constructor performs a /// dynamic (GC) allocation to do so. - Value(path_t, SourcePath const & path) : internalType(tString), _string_pad(0) + Value(path_t, SourcePath const & path) { auto block = gcAllocType(); *block = {.content = gcCopyStringIfNeeded(path.canonical().abs()), .context = String::path}; - _string = block; + raw = tag(tString, block); } /// Constructs a nix language value of type "list", with element array @@ -449,7 +481,7 @@ public: /// smaller, the list is stored inline, and the Value pointers in /// @ref items are shallow copied into this structure, without dynamically /// allocating memory. - Value(list_t, const List * items) : internalType(tList), _list(items), _list_pad(0) {} + Value(list_t, const List * items) : raw(tag(tList, items)) {} /// Constructs a nix language value of type "list", with an element array /// initialized by applying @ref transformer to each element in @ref items. @@ -464,7 +496,6 @@ public: > Value(list_t, SizedIterableT & items, TransformerT const & transformer) { - this->internalType = tList; auto list = reinterpret_cast(gcAllocBytes(sizeof(List) + items.size() * sizeof(Value *))); list->size = items.size(); @@ -472,22 +503,18 @@ public: for (size_t i = 0; i < items.size(); i++, it++) { list->elems[i] = transformer(*it); } - _list = list; + raw = tag(tList, list); } /// Constructs a nix language value of the singleton type "null". - Value(null_t) : Value(NULL_ACB) {} + Value(null_t) : raw(tag(tAuxiliary, &NULL_ACB)) {} /// Constructs a nix language value of type "set", with the attribute /// bindings pointed to by @ref bindings. /// /// The bindings are not not copied; this constructor assumes @ref bindings /// has already been suitably allocated by something like nix::buildBindings. - Value(attrs_t, Bindings * bindings) - : internalType(tAttrs) - , _attrs(bindings) - , _attrs_pad(0) - { } + Value(attrs_t, Bindings * bindings) : raw(tag(tAttrs, bindings)) {} /// Constructs a nix language lazy delayed computation, or "thunk". /// @@ -511,13 +538,11 @@ public: /// Constructs a nix language value of type "external", which is only used /// by plugins. Do any existing plugins even use this mechanism? Value(external_t, ExternalValueBase & external) - : internalType(tAuxiliary) - , _aux_pad(0) { - auto ep = gcAllocType(); - ep->type = Acb::tExternal; - ep->external = &external; - _auxiliary = ep; + auto ext = gcAllocType(); + ext->type = Acb::tExternal; + ext->external = &external; + raw = tag(tAuxiliary, ext); } /// Constructs a nix language value of type "lambda", which represents a @@ -529,22 +554,15 @@ public: Value(lambda_t, EvalMemory & mem, Env & env, ExprLambda & lambda); /// Constructs an evil thunk, whose evaluation represents infinite recursion. - explicit Value(blackhole_t) : internalType(tThunk), _thunk(&blackHole), _thunk_pad(0) {} + explicit Value(blackhole_t) : raw(tag(tThunk, &blackHole)) {} - explicit Value(const Acb & backing) - : internalType(tAuxiliary) - , _auxiliary(&backing) - , _aux_pad(0) - { - } + explicit Value(const Acb & backing) : raw(tag(tAuxiliary, &backing)) {} Value(Value const & rhs) = default; /// Move constructor. Does the same thing as the copy constructor, but /// also zeroes out the other Value. - Value(Value && rhs) - : internalType(rhs.internalType) - , _empty{ 0, 0 } + Value(Value && rhs) : raw(0) { *this = std::move(rhs); } @@ -559,9 +577,7 @@ public: *this = static_cast(rhs); if (this != &rhs) { // Kill `rhs`, because non-destructive move lol. - rhs.internalType = static_cast(0); - rhs._empty[0] = 0; - rhs._empty[1] = 0; + rhs.raw = 0; } return *this; } @@ -573,28 +589,34 @@ public: // needed by callers into methods of this type // type() == nThunk - inline bool isThunk() const { return internalType == tThunk; }; - inline bool isApp() const { return internalType == tApp; }; + inline bool isThunk() const + { + return internalType() == tThunk; + }; + inline bool isApp() const + { + return internalType() == tApp; + } inline bool isBlackhole() const { - return internalType == tThunk && _thunk == &blackHole; + return internalType() == tThunk && untag() == &blackHole; } // type() == nFunction inline bool isLambda() const { - return internalType == tAuxiliary && _auxiliary->type == Acb::tLambda; + return internalType() == tAuxiliary && auxiliary()->type == Acb::tLambda; }; inline bool isPrimOp() const { - return internalType == tAuxiliary && _auxiliary->type == Acb::tPrimOp; + return internalType() == tAuxiliary && auxiliary()->type == Acb::tPrimOp; } inline bool isPrimOpApp() const { - return internalType == tApp && app().target()->isPrimOp(); + return internalType() == tApp && app().target()->isPrimOp(); } - struct List + struct alignas(TAG_ALIGN) List { size_t size; Value * elems[0]; @@ -627,7 +649,7 @@ public: * For canonicity, the store paths should be in sorted order. */ - struct String + struct alignas(TAG_ALIGN) String { /// marker location for paths, to be used as path context. static inline const char * path[] = {"\1", nullptr}; @@ -641,7 +663,7 @@ public: } }; - struct App + struct alignas(TAG_ALIGN) App { Value * _left; size_t _n; @@ -670,7 +692,7 @@ public: /// auxiliary control block for values that require more space. /// these blocks are usually heap-allocated in GC memory space. - struct Acb + struct alignas(TAG_ALIGN) Acb { enum { tExternal, @@ -705,52 +727,12 @@ public: Env * env; ExprLambda * fun; }; - struct Thunk + struct alignas(TAG_ALIGN) Thunk { Env * env; Expr * expr; }; - union - { - /// Dummy field, which takes up as much space as the largest union variants - /// to set the union's memory to zeroed memory. - uintptr_t _empty[2]; - - NixInt _integer; - struct { - bool _boolean; - uintptr_t _bool_pad; - }; - - struct - { - const String * _string; - uintptr_t _string_pad; - }; - struct { - Bindings * _attrs; - uintptr_t _attrs_pad; - }; - struct { - const List * _list; - uintptr_t _list_pad; - }; - struct { - Thunk * _thunk; - uintptr_t _thunk_pad; - }; - struct - { - App * _app; - uintptr_t _app_pad; - }; - struct { - const Acb * _auxiliary; - uintptr_t _aux_pad; - }; - }; - /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd @@ -760,48 +742,42 @@ public: */ inline ValueType type(bool invalidIsThunk = false) const { - switch (internalType) { - case tInt: return nInt; - case tBool: return nBool; - case tString: - return _string->isPath() ? nPath : nString; - case tAttrs: return nAttrs; - case tList: - return nList; - case tAuxiliary: - switch (_auxiliary->type) { - case Acb::tExternal: - return nExternal; - case Acb::tFloat: - return nFloat; - case Acb::tNull: - return nNull; - case Acb::tPrimOp: - case Acb::tLambda: - return nFunction; - case Acb::tInt: - return nInt; - } + switch (internalType()) { + case tInt: + return nInt; + case tBool: + return nBool; + case tString: + return untag()->isPath() ? nPath : nString; + case tAttrs: + return nAttrs; + case tList: + return nList; + case tAuxiliary: + switch (untag()->type) { + case Acb::tExternal: + return nExternal; + case Acb::tFloat: + return nFloat; + case Acb::tNull: + return nNull; + case Acb::tPrimOp: + case Acb::tLambda: + return nFunction; + case Acb::tInt: + return nInt; + } case tThunk: return nThunk; case tApp: return app().target()->isPrimOp() ? nFunction : nThunk; - } + } if (invalidIsThunk) return nThunk; else abort(); } - /** - * After overwriting an app node, be sure to clear pointers in the - * Value to ensure that the target isn't kept alive unnecessarily. - */ - inline void clearValue() - { - _empty[0] = _empty[1] = 0; - } - inline void mkInt(NixInt::Inner n) { mkInt(NixInt{n}); @@ -814,17 +790,14 @@ public: inline void mkBool(bool b) { - clearValue(); - internalType = tBool; - _boolean = b; + raw = tag(tBool, b); } inline void mkString(const char * s, const char * * context = 0) { - internalType = tString; auto block = gcAllocType(); *block = {.content = s, .context = context}; - _string = block; + raw = tag(tString, block); } void mkString(std::string_view s); @@ -837,11 +810,9 @@ public: inline void mkPath(const char * path) { - clearValue(); - internalType = tString; auto block = gcAllocType(); *block = {.content = path, .context = String::path}; - _string = block; + raw = tag(tString, block); } inline void mkNull() @@ -851,9 +822,7 @@ public: inline void mkAttrs(Bindings * a) { - clearValue(); - internalType = tAttrs; - _attrs = a; + raw = tag(tAttrs, a); } Value & mkAttrs(BindingsBuilder & bindings); @@ -872,17 +841,17 @@ public: bool isList() const { - return internalType == tList; + return internalType() == tList; } Value * const * listElems() const { - return _list->elems; + return untag()->elems; } size_t listSize() const { - return _list->size; + return untag()->size; } /** @@ -922,83 +891,84 @@ public: SourcePath path() const { - assert(internalType == tString && _string->isPath()); - return SourcePath{CanonPath(_string->content)}; + assert(internalType() == tString && untag()->isPath()); + return SourcePath{CanonPath(untag()->content)}; } std::string_view str() const { - assert(internalType == tString && !_string->isPath()); - return std::string_view(_string->content); + assert(internalType() == tString && !untag()->isPath()); + return std::string_view(untag()->content); } NixInt integer() const { - if (internalType == tInt) { - return _integer; + if (internalType() == tInt) { + intptr_t tmp; + memcpy(&tmp, &raw, sizeof(tmp)); + return NixInt(tmp >> 3); } else { - assert(internalType == tAuxiliary && _auxiliary->type == Acb::tInt); - return static_cast(_auxiliary)->value; + assert(internalType() == tAuxiliary && untag()->type == Acb::tInt); + return untag()->value; } } bool boolean() const { - return _boolean; + return untag(); } const auto & string() const { - return *_string; + return *untag(); } auto attrs() const { - return _attrs; + return untag(); } const auto & thunk() const { - return *_thunk; + return *untag(); } App & app() { - return *_app; + return *untag(); } const App & app() const { - return *_app; + return *untag(); } const auto & lambda() const { - assert(internalType == tAuxiliary && _auxiliary->type == Acb::tLambda); - return *static_cast(_auxiliary); + return *untag(); } const PrimOp * primOp() const { - assert(internalType == tAuxiliary && _auxiliary->type == Acb::tPrimOp); - return static_cast(_auxiliary); + assert(internalType() == tAuxiliary && untag()->type == Acb::tPrimOp); + return untag(); } const ExternalValueBase * external() const { - assert(internalType == tAuxiliary && _auxiliary->type == Acb::tExternal); - return static_cast(_auxiliary)->external; + assert(internalType() == tAuxiliary && untag()->type == Acb::tExternal); + return untag()->external; } NixFloat fpoint() const { - assert(internalType == tAuxiliary && _auxiliary->type == Acb::tFloat); - return static_cast(_auxiliary)->value; + assert(internalType() == tAuxiliary && untag()->type == Acb::tFloat); + return untag()->value; } const Acb * auxiliary() const { - return _auxiliary; + return untag(); } };