libexpr: AST-dump non-utf8 strings as byte arrays

our strings need not be utf8. json requires utf. -sigh-

fixes #1052

Change-Id: I50ecd9882252370bb81845b099b11a7190475d48

lix/libexpr/nixexpr.cc

@@ -35,6 +35,24 @@ AttrName::AttrName(PosIdx pos, std::unique_ptr<Expr> e) : pos(pos), expr(std::mo
 {
 }
 
+static JSON stringToJSON(std::string_view s)
+{
+    try {
+        JSON value = s;
+        // dump it now to catch invalid utf8 strings early. this code path is not
+        // hot, so the extra memory allocation and encoding is not worth avoiding
+        (void) value.dump();
+        return value;
+    } catch (nlohmann::json::type_error & e) { // NOLINT(lix-foreign-exceptions)
+        if (e.id == 316) {
+            // invalid utf8 in string! serialize as byte array instead
+            return s | std::ranges::to<std::vector<unsigned char>>();
+        } else {
+            throw; // NOLINT(lix-foreign-exceptions)
+        }
+    }
+}
+
 JSON Expr::toJSON(const SymbolTable & symbols) const
 {
     abort();
@@ -57,7 +75,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const
             break;
         case nString:
             valueType = "String";
-            value = v.str();
+            value = stringToJSON(v.str());
             break;
         case nPath:
             valueType = "Path";
@@ -77,10 +95,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const
 
 JSON ExprVar::toJSON(const SymbolTable & symbols) const
 {
-    return {
-        {"_type", "ExprVar"},
-        {"value", symbols[name]}
-    };
+    return {{"_type", "ExprVar"}, {"value", stringToJSON(symbols[name])}};
 }
 
 JSON ExprInheritFrom::toJSON(SymbolTable const & symbols) const
@@ -126,11 +141,19 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
-            out["attrs"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols);
-            break;
-        case AttrDef::Kind::Inherited:
-            out["inherit"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols);
+        case AttrDef::Kind::Inherited: {
+            const auto key = i->second.kind == AttrDef::Kind::Plain ? "attrs" : "inherit";
+            auto name = stringToJSON(symbols[i->first]);
+            if (name.is_string()) {
+                out[key][name] = i->second.e->toJSON(symbols);
+            } else {
+                out[fmt("binary_%s", key)][key].push_back({
+                    {"name", name},
+                    {"value", i->second.e->toJSON(symbols)},
+                });
+            }
             break;
+        }
         case AttrDef::Kind::InheritedFrom: {
             auto & select = i->second.e->cast<ExprSelect>();
             auto & from = select.e->cast<ExprInheritFrom>();
@@ -150,7 +173,7 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
     for (const auto & [from, syms] : inheritsFrom) {
         JSON attrs = JSON::array();
         for (auto sym : syms)
-            attrs.push_back(symbols[sym]);
+            attrs.push_back(stringToJSON(symbols[sym]));
         out["inheritFrom"].push_back({
             {"from", inheritFromExprs[from]->toJSON(symbols)},
             {"attrs", attrs}
@@ -188,11 +211,13 @@ JSON ExprList::toJSON(const SymbolTable & symbols) const
 
 void SimplePattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
 {
+    // name must be alphanumeric
     out["arg"] = symbols[name];
 }
 
 void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
 {
+    // name must be alphanumeric
     if (name)
         out["arg"] = symbols[name];
 
@@ -200,6 +225,7 @@ void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) co
     // same expression being printed in two different ways depending on its
     // context. always use lexicographic ordering to avoid this.
     for (const Formal & i : lexicographicOrder(symbols)) {
+        // names must be alphanumeric
         if (i.def)
             out["formals"][std::string(symbols[i.name])] = i.def->toJSON(symbols);
         else
@@ -313,7 +339,7 @@ JSON printAttrPathToJson(const SymbolTable & symbols, const AttrPath & attrPath)
     JSON out = JSON::array();
     for (auto & i : attrPath) {
         if (i.symbol)
-            out.push_back(symbols[i.symbol]);
+            out.push_back(stringToJSON(symbols[i.symbol]));
         else
             out.push_back(i.expr->toJSON(symbols));
     }

tests/functional2/lang/utf8/in-invalid-utf8-string.nix

@@ -0,0 +1,9 @@
+# parser tests serialize the AST as json, and json does not like non-utf8 strings at all
+{
+  invalidString = "välid bit ÿ moré vÃ¥lid";
+  invalidAttrPath = x: x.a."aÿb".c;
+  "invalidÿName" = 1;
+  invalidInherit = x: with x; { inherit "aÿb"; };
+  invalidInheritFrom = x: { inherit (x) "aÿb"; };
+  invalidLet = let "aÿb" = 1; in 1;
+}

tests/functional2/lang/utf8/parse-okay-invalid-utf8-string.out.exp

@@ -0,0 +1,118 @@
+_type: ExprSet
+attrs:
+  invalidAttrPath:
+    _type: ExprLambda
+    arg: x
+    body:
+      _type: ExprSelect
+      attrs:
+        - a
+        - - 97
+          - 255
+          - 98
+        - c
+      e:
+        _type: ExprVar
+        value: x
+  invalidInherit:
+    _type: ExprLambda
+    arg: x
+    body:
+      _type: ExprWith
+      attrs:
+        _type: ExprVar
+        value: x
+      body:
+        _type: ExprSet
+        binary_inherit:
+          inherit:
+            - name:
+                - 97
+                - 255
+                - 98
+              value:
+                _type: ExprVar
+                value:
+                  - 97
+                  - 255
+                  - 98
+        recursive: false
+  invalidInheritFrom:
+    _type: ExprLambda
+    arg: x
+    body:
+      _type: ExprSet
+      inheritFrom:
+        - attrs:
+            - - 97
+              - 255
+              - 98
+          from:
+            _type: ExprVar
+            value: x
+      recursive: false
+  invalidLet:
+    _type: ExprLet
+    binary_attrs:
+      attrs:
+        - name:
+            - 97
+            - 255
+            - 98
+          value:
+            _type: ExprLiteral
+            value: 1
+            valueType: Int
+    body:
+      _type: ExprLiteral
+      value: 1
+      valueType: Int
+  invalidString:
+    _type: ExprLiteral
+    value:
+      - 118
+      - 195
+      - 164
+      - 108
+      - 105
+      - 100
+      - 32
+      - 98
+      - 105
+      - 116
+      - 32
+      - 255
+      - 32
+      - 109
+      - 111
+      - 114
+      - 195
+      - 169
+      - 32
+      - 118
+      - 195
+      - 165
+      - 108
+      - 105
+      - 100
+    valueType: String
+binary_attrs:
+  attrs:
+    - name:
+        - 105
+        - 110
+        - 118
+        - 97
+        - 108
+        - 105
+        - 100
+        - 255
+        - 78
+        - 97
+        - 109
+        - 101
+      value:
+        _type: ExprLiteral
+        value: 1
+        valueType: Int
+recursive: false

tests/functional2/lang/utf8/test.toml

@@ -0,0 +1,6 @@
+[[test]]
+runner = "parse-fail"
+
+[[test]]
+runner = "parse-okay"
+in = "in-invalid-utf8-string.nix"