diff --git a/lix/libexpr/nixexpr.cc b/lix/libexpr/nixexpr.cc index abfb629d8..61f45baf5 100644 --- a/lix/libexpr/nixexpr.cc +++ b/lix/libexpr/nixexpr.cc @@ -35,6 +35,24 @@ AttrName::AttrName(PosIdx pos, std::unique_ptr e) : pos(pos), expr(std::mo { } +static JSON stringToJSON(std::string_view s) +{ + try { + JSON value = s; + // dump it now to catch invalid utf8 strings early. this code path is not + // hot, so the extra memory allocation and encoding is not worth avoiding + (void) value.dump(); + return value; + } catch (nlohmann::json::type_error & e) { // NOLINT(lix-foreign-exceptions) + if (e.id == 316) { + // invalid utf8 in string! serialize as byte array instead + return s | std::ranges::to>(); + } else { + throw; // NOLINT(lix-foreign-exceptions) + } + } +} + JSON Expr::toJSON(const SymbolTable & symbols) const { abort(); @@ -57,7 +75,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const break; case nString: valueType = "String"; - value = v.str(); + value = stringToJSON(v.str()); break; case nPath: valueType = "Path"; @@ -77,10 +95,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const JSON ExprVar::toJSON(const SymbolTable & symbols) const { - return { - {"_type", "ExprVar"}, - {"value", symbols[name]} - }; + return {{"_type", "ExprVar"}, {"value", stringToJSON(symbols[name])}}; } JSON ExprInheritFrom::toJSON(SymbolTable const & symbols) const @@ -126,11 +141,19 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const for (auto & i : sorted) { switch (i->second.kind) { case AttrDef::Kind::Plain: - out["attrs"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols); - break; - case AttrDef::Kind::Inherited: - out["inherit"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols); + case AttrDef::Kind::Inherited: { + const auto key = i->second.kind == AttrDef::Kind::Plain ? "attrs" : "inherit"; + auto name = stringToJSON(symbols[i->first]); + if (name.is_string()) { + out[key][name] = i->second.e->toJSON(symbols); + } else { + out[fmt("binary_%s", key)][key].push_back({ + {"name", name}, + {"value", i->second.e->toJSON(symbols)}, + }); + } break; + } case AttrDef::Kind::InheritedFrom: { auto & select = i->second.e->cast(); auto & from = select.e->cast(); @@ -150,7 +173,7 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const for (const auto & [from, syms] : inheritsFrom) { JSON attrs = JSON::array(); for (auto sym : syms) - attrs.push_back(symbols[sym]); + attrs.push_back(stringToJSON(symbols[sym])); out["inheritFrom"].push_back({ {"from", inheritFromExprs[from]->toJSON(symbols)}, {"attrs", attrs} @@ -188,11 +211,13 @@ JSON ExprList::toJSON(const SymbolTable & symbols) const void SimplePattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const { + // name must be alphanumeric out["arg"] = symbols[name]; } void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const { + // name must be alphanumeric if (name) out["arg"] = symbols[name]; @@ -200,6 +225,7 @@ void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) co // same expression being printed in two different ways depending on its // context. always use lexicographic ordering to avoid this. for (const Formal & i : lexicographicOrder(symbols)) { + // names must be alphanumeric if (i.def) out["formals"][std::string(symbols[i.name])] = i.def->toJSON(symbols); else @@ -313,7 +339,7 @@ JSON printAttrPathToJson(const SymbolTable & symbols, const AttrPath & attrPath) JSON out = JSON::array(); for (auto & i : attrPath) { if (i.symbol) - out.push_back(symbols[i.symbol]); + out.push_back(stringToJSON(symbols[i.symbol])); else out.push_back(i.expr->toJSON(symbols)); } diff --git a/tests/functional2/lang/utf8/in-invalid-utf8-string.nix b/tests/functional2/lang/utf8/in-invalid-utf8-string.nix new file mode 100644 index 000000000..2459e1bd4 --- /dev/null +++ b/tests/functional2/lang/utf8/in-invalid-utf8-string.nix @@ -0,0 +1,9 @@ +# parser tests serialize the AST as json, and json does not like non-utf8 strings at all +{ + invalidString = "välid bit ˙ morĂ© vĂĄlid"; + invalidAttrPath = x: x.a."a˙b".c; + "invalid˙Name" = 1; + invalidInherit = x: with x; { inherit "a˙b"; }; + invalidInheritFrom = x: { inherit (x) "a˙b"; }; + invalidLet = let "a˙b" = 1; in 1; +} diff --git a/tests/functional2/lang/utf8/parse-okay-invalid-utf8-string.out.exp b/tests/functional2/lang/utf8/parse-okay-invalid-utf8-string.out.exp new file mode 100644 index 000000000..adb6eafc0 --- /dev/null +++ b/tests/functional2/lang/utf8/parse-okay-invalid-utf8-string.out.exp @@ -0,0 +1,118 @@ +_type: ExprSet +attrs: + invalidAttrPath: + _type: ExprLambda + arg: x + body: + _type: ExprSelect + attrs: + - a + - - 97 + - 255 + - 98 + - c + e: + _type: ExprVar + value: x + invalidInherit: + _type: ExprLambda + arg: x + body: + _type: ExprWith + attrs: + _type: ExprVar + value: x + body: + _type: ExprSet + binary_inherit: + inherit: + - name: + - 97 + - 255 + - 98 + value: + _type: ExprVar + value: + - 97 + - 255 + - 98 + recursive: false + invalidInheritFrom: + _type: ExprLambda + arg: x + body: + _type: ExprSet + inheritFrom: + - attrs: + - - 97 + - 255 + - 98 + from: + _type: ExprVar + value: x + recursive: false + invalidLet: + _type: ExprLet + binary_attrs: + attrs: + - name: + - 97 + - 255 + - 98 + value: + _type: ExprLiteral + value: 1 + valueType: Int + body: + _type: ExprLiteral + value: 1 + valueType: Int + invalidString: + _type: ExprLiteral + value: + - 118 + - 195 + - 164 + - 108 + - 105 + - 100 + - 32 + - 98 + - 105 + - 116 + - 32 + - 255 + - 32 + - 109 + - 111 + - 114 + - 195 + - 169 + - 32 + - 118 + - 195 + - 165 + - 108 + - 105 + - 100 + valueType: String +binary_attrs: + attrs: + - name: + - 105 + - 110 + - 118 + - 97 + - 108 + - 105 + - 100 + - 255 + - 78 + - 97 + - 109 + - 101 + value: + _type: ExprLiteral + value: 1 + valueType: Int +recursive: false diff --git a/tests/functional2/lang/utf8/test.toml b/tests/functional2/lang/utf8/test.toml new file mode 100644 index 000000000..1c1c9d8ff --- /dev/null +++ b/tests/functional2/lang/utf8/test.toml @@ -0,0 +1,6 @@ +[[test]] +runner = "parse-fail" + +[[test]] +runner = "parse-okay" +in = "in-invalid-utf8-string.nix"