From 7e68f93ed77e0ced764af3ee009ab4d152387ef8 Mon Sep 17 00:00:00 2001 From: piegames Date: Sun, 14 Dec 2025 20:54:28 +0100 Subject: [PATCH] libexpr: Require whitespace between certain tokens Fixes #135, #136 Co-authored-by: Commentator2.0 Change-Id: Ia1880633c1ee3b9242487fbc30b6d781d88987fb --- doc/manual/rl-next/deprecated-features.md | 1 + lix/libexpr/parser/grammar.hh | 45 ++++++++--- lix/libexpr/parser/parser-impl1.inc.cc | 22 +++++ lix/libexpr/parser/state.hh | 14 ++++ .../tokens-no-whitespace.md | 18 +++++ lix/libutil/meson.build | 1 + .../((0)(0))-depr.out.exp | 22 +++++ .../parser-token-whitespace/((0)(0)).out.exp | 22 +++++ .../((a).0)-depr.err.exp | 4 + .../((a).0)-depr.out.exp | 21 +++++ .../parser-token-whitespace/((a).0).err.exp | 4 + .../parser-token-whitespace/((a).0).out.exp | 21 +++++ .../((a).a)-depr.out.exp | 17 ++++ .../parser-token-whitespace/((a).a).out.exp | 17 ++++ .../(0.\"\")-depr.out.exp" | 18 +++++ .../parser-token-whitespace/(0.\"\").err.exp" | 4 + .../parser-token-whitespace/(0.)-depr.err.exp | 4 + .../lang/parser-token-whitespace/(0.).err.exp | 4 + .../(0.0.0)-depr.err.exp | 4 + .../(0.0.0)-depr.out.exp | 22 +++++ .../parser-token-whitespace/(0.0.0).err.exp | 4 + .../(0.a)-depr.out.exp | 18 +++++ .../parser-token-whitespace/(0.a).err.exp | 4 + .../(00.)-depr.err.exp | 4 + .../parser-token-whitespace/(00.).err.exp | 4 + .../(00012.3)-depr.err.exp | 4 + .../(00012.3)-depr.out.exp | 22 +++++ .../parser-token-whitespace/(00012.3).err.exp | 4 + .../parser-token-whitespace/(0a)-depr.out.exp | 21 +++++ .../lang/parser-token-whitespace/(0a).err.exp | 4 + .../(0https:--a)-depr.out.exp | 22 +++++ .../(0https:--a).err.exp | 4 + .../(0x10)-depr.out.exp | 21 +++++ .../parser-token-whitespace/(0x10).err.exp | 4 + .../(1.\"\")-depr.out.exp" | 22 +++++ .../parser-token-whitespace/(1.\"\").err.exp" | 4 + .../parser-token-whitespace/(1.)-depr.out.exp | 12 +++ .../lang/parser-token-whitespace/(1.).out.exp | 12 +++ .../(1.a)-depr.out.exp | 21 +++++ .../parser-token-whitespace/(1.a).err.exp | 4 + .../(a(\"\"))-depr.out.exp" | 21 +++++ .../(a(\"\")).out.exp" | 21 +++++ .../(foo\"1\"2)-depr.out.exp" | 26 ++++++ .../(foo\"1\"2).err.exp" | 4 + .../[(0)(0)]-depr.out.exp | 22 +++++ .../parser-token-whitespace/[(0)(0)].out.exp | 22 +++++ .../[(a).0]-depr.err.exp | 4 + .../[(a).0]-depr.out.exp | 21 +++++ .../parser-token-whitespace/[(a).0].err.exp | 4 + .../parser-token-whitespace/[(a).0].out.exp | 21 +++++ .../[(a).a]-depr.out.exp | 22 +++++ .../parser-token-whitespace/[(a).a].out.exp | 22 +++++ .../[0.\"\"]-depr.out.exp" | 23 ++++++ .../parser-token-whitespace/[0.\"\"].err.exp" | 4 + .../[0.0.0]-depr.err.exp | 4 + .../[0.0.0]-depr.out.exp | 22 +++++ .../parser-token-whitespace/[0.0.0].err.exp | 4 + .../parser-token-whitespace/[0.]-depr.err.exp | 4 + .../lang/parser-token-whitespace/[0.].err.exp | 4 + .../[0.a]-depr.out.exp | 23 ++++++ .../parser-token-whitespace/[0.a].err.exp | 4 + .../[00.]-depr.err.exp | 4 + .../parser-token-whitespace/[00.].err.exp | 4 + .../[00012.3]-depr.err.exp | 4 + .../[00012.3]-depr.out.exp | 22 +++++ .../parser-token-whitespace/[00012.3].err.exp | 4 + .../parser-token-whitespace/[0a]-depr.out.exp | 21 +++++ .../lang/parser-token-whitespace/[0a].err.exp | 4 + .../[0https:--a]-depr.out.exp | 22 +++++ .../[0https:--a].err.exp | 4 + .../[0x10]-depr.out.exp | 21 +++++ .../parser-token-whitespace/[0x10].err.exp | 4 + .../[1.\"\"]-depr.out.exp" | 22 +++++ .../parser-token-whitespace/[1.\"\"].err.exp" | 4 + .../parser-token-whitespace/[1.]-depr.out.exp | 17 ++++ .../lang/parser-token-whitespace/[1.].out.exp | 17 ++++ .../[1.a]-depr.out.exp | 21 +++++ .../parser-token-whitespace/[1.a].err.exp | 4 + .../[a(\"\")]-depr.out.exp" | 21 +++++ .../[a(\"\")].out.exp" | 21 +++++ .../[foo\"1\"2]-depr.out.exp" | 26 ++++++ .../[foo\"1\"2].err.exp" | 4 + .../lang/parser-token-whitespace/__init__.py | 0 .../test_whitespace_things.py | 80 +++++++++++++++++++ tests/functional2/lang/set/test.toml | 8 ++ tests/nixpkgs/eval.nix | 1 + tests/nixpkgs/lib.nix | 1 + 87 files changed, 1152 insertions(+), 11 deletions(-) create mode 100644 lix/libutil/deprecated-features/tokens-no-whitespace.md create mode 100644 tests/functional2/lang/parser-token-whitespace/((0)(0))-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((0)(0)).out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).0)-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).0)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).0).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).0).out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).a)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/((a).a).out.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/(0.\"\")-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/(0.\"\").err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.)-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.0.0)-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.0.0)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.0.0).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.a)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0.a).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(00.)-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(00.).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(00012.3)-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(00012.3)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(00012.3).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0a)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0a).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0https:--a)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0https:--a).err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0x10)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(0x10).err.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/(1.\"\")-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/(1.\"\").err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/(1.)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(1.).out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(1.a)-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/(1.a).err.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/(a(\"\"))-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/(a(\"\")).out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/(foo\"1\"2)-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/(foo\"1\"2).err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/[(0)(0)]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(0)(0)].out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).0]-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).0]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).0].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).0].out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).a]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[(a).a].out.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/[0.\"\"]-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/[0.\"\"].err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.0.0]-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.0.0]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.0.0].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.]-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.a]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0.a].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[00.]-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[00.].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[00012.3]-depr.err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[00012.3]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[00012.3].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0a]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0a].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0https:--a]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0https:--a].err.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0x10]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[0x10].err.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/[1.\"\"]-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/[1.\"\"].err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/[1.]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[1.].out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[1.a]-depr.out.exp create mode 100644 tests/functional2/lang/parser-token-whitespace/[1.a].err.exp create mode 100644 "tests/functional2/lang/parser-token-whitespace/[a(\"\")]-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/[a(\"\")].out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/[foo\"1\"2]-depr.out.exp" create mode 100644 "tests/functional2/lang/parser-token-whitespace/[foo\"1\"2].err.exp" create mode 100644 tests/functional2/lang/parser-token-whitespace/__init__.py create mode 100644 tests/functional2/lang/parser-token-whitespace/test_whitespace_things.py create mode 100644 tests/functional2/lang/set/test.toml diff --git a/doc/manual/rl-next/deprecated-features.md b/doc/manual/rl-next/deprecated-features.md index 0a72fc82c..b54eae104 100644 --- a/doc/manual/rl-next/deprecated-features.md +++ b/doc/manual/rl-next/deprecated-features.md @@ -14,3 +14,4 @@ You can opt in into the old behavior with `--extra-deprecated-features` or any e - `rec-set-merges` Attribute sets like `{ foo = {}; foo.bar = 42;}` implicitly merge at parse time, however if one of them is marked as recursive but not the others then the recursive attribute may get lost (order-dependent). Therefore, merging attrs with mixed-`rec` is now forbidden. - `rec-set-dynamic-attrs` Dynamic attributes have weird semantics in the presence of recursive attrsets (they evaluate *after* the rest of the set). This is now forbidden. - `or-as-identifier` `or` as an identifier has always been weird since the `or` (almost-)keyword has been introduced. We are deprecating the backcompat hacks from the early days of Nix in favor of making `or` a full and proper keyword. +- `tokens-no-whitespace` Function applications without space around the arguments like `0a`, `0.00.0` or `foo"1"2` are now forbidden. The same applies to list elements. The primary reason for this deprecation is to remove foot guns around surprising tokenization rules regarding number literals, but this will also free up some syntax for other purposes (e.g. `r""` strings) for reuse at some point in the future. diff --git a/lix/libexpr/parser/grammar.hh b/lix/libexpr/parser/grammar.hh index 37959fb4f..1694fa204 100644 --- a/lix/libexpr/parser/grammar.hh +++ b/lix/libexpr/parser/grammar.hh @@ -12,6 +12,8 @@ // eolf rules in favor of reproducing the old flex lexer as faithfully as // possible, and deferring calculation of positions to downstream users. +/* clang-format absolutely butchers the template arguments */ +// clang-format off namespace nix::parser::grammar::v1 { using namespace tao::pegtl; @@ -154,13 +156,20 @@ struct identifier : _not_at_any_keyword< star > {}; +struct _integer { + // Add a special case to specifically catch and forbid `0.a`, which otherwise parses as selection + // TODO this is only an artifact of `0.` not being a float, and should be eliminated once that is corrected + struct at_dot_id : success {}; +}; + // floats may extend ints, thus these rules are very similar. -struct integer : seq< +struct integer : _integer, seq< sor< seq, star, not_at>>, seq, not_at, digit>, star> >, - not_at<_extend_as_path> + not_at<_extend_as_path>, + opt, sor>>>, _integer::at_dot_id> // This will be not_at in the future once it becomes a hard error > {}; struct _floating { @@ -169,6 +178,7 @@ struct _floating { struct floating : _floating, seq< sor< seq, star, one<'.'>, star>, + // TODO once the deprecations don't have an opt-out anymore, parse `0.` like `1.` by using `star` seq, one<'.'>, plus>, _floating::no_leading_zero >, @@ -536,14 +546,26 @@ struct _expr { must> > {}; - struct _simple : sor< - id, - int_, - float_, - string, - ind_string, - path, - uri, + struct _simple { + struct noseps : semantic, success {}; + // the subset of `simple` that may not directly follow each other without whitespace as a token boundary + struct token_simple : sor< + id, + int_, + float_, + string, + ind_string, + path, + uri + > { }; + }; + + struct simple : _simple, sor< + seq< + _simple::token_simple, + // error if we have without whitespace in-between + opt, _simple::noseps> + >, seq, seps, must, seps, must>>, ancient_let, rec_set, @@ -552,7 +574,7 @@ struct _expr { > {}; struct _select { - struct head : _simple {}; + struct head : simple {}; struct attr : semantic, seq {}; struct attr_or : semantic, must