nette-neon 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nette_neon-1.0.0/.gitignore +6 -0
- nette_neon-1.0.0/PKG-INFO +249 -0
- nette_neon-1.0.0/docs/internals.md +89 -0
- nette_neon-1.0.0/docs/skills/agents.md +65 -0
- nette_neon-1.0.0/examples/config-editor/editor.py +69 -0
- nette_neon-1.0.0/examples/config-editor/readme.md +36 -0
- nette_neon-1.0.0/examples/edge-cases/edges.py +40 -0
- nette_neon-1.0.0/examples/edge-cases/readme.md +44 -0
- nette_neon-1.0.0/examples/editing/adding.py +41 -0
- nette_neon-1.0.0/examples/editing/inline.py +33 -0
- nette_neon-1.0.0/examples/editing/readme.md +138 -0
- nette_neon-1.0.0/examples/editing/removing.py +62 -0
- nette_neon-1.0.0/examples/editing/structures.py +35 -0
- nette_neon-1.0.0/examples/editing/values.py +35 -0
- nette_neon-1.0.0/examples/encoding/encode.py +41 -0
- nette_neon-1.0.0/examples/encoding/readme.md +82 -0
- nette_neon-1.0.0/examples/linting/lint.py +32 -0
- nette_neon-1.0.0/examples/linting/readme.md +41 -0
- nette_neon-1.0.0/examples/parsing/decode.py +40 -0
- nette_neon-1.0.0/examples/parsing/readme.md +53 -0
- nette_neon-1.0.0/examples/parsing/roundtrip.py +18 -0
- nette_neon-1.0.0/examples/readme.md +18 -0
- nette_neon-1.0.0/examples/tree/navigate.py +45 -0
- nette_neon-1.0.0/examples/tree/readme.md +73 -0
- nette_neon-1.0.0/examples/tree/tokens.py +26 -0
- nette_neon-1.0.0/examples/verify.py +73 -0
- nette_neon-1.0.0/license.md +21 -0
- nette_neon-1.0.0/pyproject.toml +52 -0
- nette_neon-1.0.0/readme.md +229 -0
- nette_neon-1.0.0/src/neon/__init__.py +83 -0
- nette_neon-1.0.0/src/neon/__main__.py +5 -0
- nette_neon-1.0.0/src/neon/_journal.py +40 -0
- nette_neon-1.0.0/src/neon/builder.py +99 -0
- nette_neon-1.0.0/src/neon/dumper.py +86 -0
- nette_neon-1.0.0/src/neon/editing.py +1547 -0
- nette_neon-1.0.0/src/neon/encoder.py +314 -0
- nette_neon-1.0.0/src/neon/errors.py +15 -0
- nette_neon-1.0.0/src/neon/facade.py +23 -0
- nette_neon-1.0.0/src/neon/layout.py +126 -0
- nette_neon-1.0.0/src/neon/lexer.py +85 -0
- nette_neon-1.0.0/src/neon/lint.py +196 -0
- nette_neon-1.0.0/src/neon/nodes.py +856 -0
- nette_neon-1.0.0/src/neon/parser.py +479 -0
- nette_neon-1.0.0/src/neon/position.py +38 -0
- nette_neon-1.0.0/src/neon/printer.py +48 -0
- nette_neon-1.0.0/src/neon/py.typed +0 -0
- nette_neon-1.0.0/src/neon/scanner.py +210 -0
- nette_neon-1.0.0/src/neon/token.py +390 -0
- nette_neon-1.0.0/src/neon/traverser.py +96 -0
- nette_neon-1.0.0/src/neon/values.py +363 -0
- nette_neon-1.0.0/tests/corpus.py +157 -0
- nette_neon-1.0.0/tests/differential.py +78 -0
- nette_neon-1.0.0/tests/fetch_spec.py +24 -0
- nette_neon-1.0.0/tests/guarantees.py +117 -0
- nette_neon-1.0.0/tests/neon-spec.lock +1 -0
- nette_neon-1.0.0/tests/smoke.py +43 -0
- nette_neon-1.0.0/tests/test_corpus.py +76 -0
- nette_neon-1.0.0/tests/test_editing.py +680 -0
- nette_neon-1.0.0/tests/test_encoder.py +228 -0
- nette_neon-1.0.0/tests/test_external.py +32 -0
- nette_neon-1.0.0/tests/test_lexer.py +153 -0
- nette_neon-1.0.0/tests/test_lint.py +118 -0
- nette_neon-1.0.0/tests/test_mutation_fuzz.py +538 -0
- nette_neon-1.0.0/tests/test_mutations.py +252 -0
- nette_neon-1.0.0/tests/test_nodes.py +226 -0
- nette_neon-1.0.0/tests/test_traverser.py +174 -0
- nette_neon-1.0.0/tests/typed.py +54 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nette-neon
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: NEON parser, encoder and lossless editor: change a value in a config file and keep its comments and formatting.
|
|
5
|
+
Project-URL: Homepage, https://neon.nette.org
|
|
6
|
+
Project-URL: Repository, https://github.com/nette/neon-py
|
|
7
|
+
Author: David Grudl
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: license.md
|
|
10
|
+
Keywords: config,cst,format-preserving,neon,nette,parser,round-trip
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.13
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
NEON for Python
|
|
22
|
+
===============
|
|
23
|
+
|
|
24
|
+
[](https://pypi.org/project/nette-neon/)
|
|
25
|
+
[](https://github.com/nette/neon-py/actions)
|
|
26
|
+
[](https://github.com/nette/neon-py/blob/master/license.md)
|
|
27
|
+
|
|
28
|
+
<!---->
|
|
29
|
+
|
|
30
|
+
<h3>
|
|
31
|
+
|
|
32
|
+
✅ A NEON parser that keeps [every comment and space](#the-tree-nodes-slots-tokens-trivia)<br>
|
|
33
|
+
✅ [Edit a configuration file](#format-preserving-editing) and the diff is only your change<br>
|
|
34
|
+
✅ [`neon-lint`](#for-editors-and-ai-agents) for editors, CI and AI agents<br>
|
|
35
|
+
✅ [No dependencies](#installation), fully typed, Python 3.13+
|
|
36
|
+
|
|
37
|
+
</h3>
|
|
38
|
+
|
|
39
|
+
<!---->
|
|
40
|
+
|
|
41
|
+
**NEON parser, encoder and lossless editor for Python.** NEON is the human-friendly
|
|
42
|
+
configuration format of the Nette framework, a YAML alternative with entities, and the format of
|
|
43
|
+
`phpstan.neon`. This library parses it into a concrete syntax tree in which every character of
|
|
44
|
+
the file has its place, so printing the tree gives the file back byte for byte:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
doc = parse(text)
|
|
48
|
+
doc.print() == text # True, for any input NEON accepts
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
That is what makes editing safe. Change a value, add a key, remove an item, and the rest of the
|
|
52
|
+
file stays as its author wrote it:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
doc.set_value(["database", "host"], "db.example.com")
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
- host: localhost # the server
|
|
60
|
+
+ host: db.example.com # the server
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
On that you build scripts that bump a setting in every `phpstan.neon` of a monorepo, installers
|
|
64
|
+
that register an extension in `config.neon`, migrations that rename a service, and tools that
|
|
65
|
+
read the configuration of a PHP project without running PHP.
|
|
66
|
+
|
|
67
|
+
The library comes from David Grudl, the author of [Nette](https://nette.org),
|
|
68
|
+
[Latte](https://latte.nette.org) and [Tracy](https://tracy.nette.org). NEON has been the
|
|
69
|
+
configuration format of Nette since 2011, read by every Nette application, and this
|
|
70
|
+
implementation follows the same specification as [nette/neon](https://github.com/nette/neon) for
|
|
71
|
+
PHP and [@nette/neon](https://github.com/nette/neon-js) for JavaScript.
|
|
72
|
+
|
|
73
|
+
<!---->
|
|
74
|
+
|
|
75
|
+
NEON in one minute
|
|
76
|
+
==================
|
|
77
|
+
|
|
78
|
+
```neon
|
|
79
|
+
# mappings and sequences, indented by tabs or spaces
|
|
80
|
+
database:
|
|
81
|
+
host: localhost
|
|
82
|
+
port: 3306
|
|
83
|
+
|
|
84
|
+
roles: [guest, member, admin] # inline arrays
|
|
85
|
+
|
|
86
|
+
# entities: a value with attributes, the way Nette writes services
|
|
87
|
+
mailer: App\Mailer(@smtp, timeout: 30)
|
|
88
|
+
|
|
89
|
+
note: '''
|
|
90
|
+
multiline strings
|
|
91
|
+
without escaping
|
|
92
|
+
'''
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
JSON is a subset of NEON, except for a colon at the start of a line glued to its value
|
|
96
|
+
(`"a"` + new line + `:1`), which NEON reads as a literal. The format is described at
|
|
97
|
+
[ne-on.org](https://ne-on.org) and in the [manual](https://doc.nette.org/en/neon/format).
|
|
98
|
+
|
|
99
|
+
▶ Runnable examples: [examples/edge-cases](examples/edge-cases/)
|
|
100
|
+
|
|
101
|
+
<!---->
|
|
102
|
+
|
|
103
|
+
Format-preserving editing
|
|
104
|
+
=========================
|
|
105
|
+
|
|
106
|
+
Most YAML libraries for Python, like PyYAML, drop comments and formatting when they write a
|
|
107
|
+
file. Here keeping them is an invariant, not an effort: every edit changes the tokens it edits,
|
|
108
|
+
and the four guarantees of the specification (the file reads back as the same tree, with the
|
|
109
|
+
expected value, with the change only where it was made) are checked after every operation of a
|
|
110
|
+
large corpus of edits.
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from neon import CommentPolicy, parse
|
|
114
|
+
|
|
115
|
+
doc = parse(text)
|
|
116
|
+
doc.set_value(["database", "port"], 3307) # 0x0CEA stays hexadecimal: 0x0CEB
|
|
117
|
+
doc.set_value(["database", "debug"], False) # yes becomes no, not false
|
|
118
|
+
doc.set_value(["timeout"], 30) # the same value: no change at all
|
|
119
|
+
doc.set_value(["parameters", "admin"], "a@b.cz") # a new key, indented like its siblings
|
|
120
|
+
doc.add_item(["services"], "App\\OrderFacade") # a new item of a list
|
|
121
|
+
doc.remove_item(["database", "driver"]) # with the comment that describes it
|
|
122
|
+
doc.remove_item(["mail"], comments=CommentPolicy.MOVE_TO_NEXT_TOKEN)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A missing key in the middle of a path is an error, so a typo does not create a new section.
|
|
126
|
+
Removing the last item of a collection leaves it empty (`services: []`), because in a
|
|
127
|
+
configuration empty and missing often mean different things; an item marked `nullable` loses its
|
|
128
|
+
value instead. A refused edit changes nothing at all.
|
|
129
|
+
|
|
130
|
+
▶ Runnable examples: [examples/editing](examples/editing/), [examples/config-editor](examples/config-editor/)
|
|
131
|
+
|
|
132
|
+
<!---->
|
|
133
|
+
|
|
134
|
+
Installation
|
|
135
|
+
============
|
|
136
|
+
|
|
137
|
+
```shell
|
|
138
|
+
uv add nette-neon # or: pip install nette-neon
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The package is imported as `neon`. Python 3.13 or newer, fully typed (`py.typed`), no
|
|
142
|
+
dependencies.
|
|
143
|
+
|
|
144
|
+
<!---->
|
|
145
|
+
|
|
146
|
+
The tree: nodes, slots, tokens, trivia
|
|
147
|
+
======================================
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
item = doc.find_item(["database", "user"])
|
|
151
|
+
item.get_value() # 'root'
|
|
152
|
+
item.get_trailing_comments() # the comment on its line
|
|
153
|
+
item.value.get_first_token().text # the text exactly as written
|
|
154
|
+
item.get_position() # Position(line, column, offset), from the current tree
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Eight node classes (`DocumentNode`, `BlockArrayNode`, `InlineArrayNode`, `ArrayItemNode`,
|
|
158
|
+
`EntityNode`, `EntityChainNode`, `StringNode`, `LiteralNode`) with named slots, tokens with their
|
|
159
|
+
trivia (whitespace, line endings, comments), `find()`, `find_ancestor()`, siblings, and
|
|
160
|
+
`Traverser` for a walk with replacements.
|
|
161
|
+
|
|
162
|
+
▶ Runnable examples: [examples/tree](examples/tree/)
|
|
163
|
+
|
|
164
|
+
<!---->
|
|
165
|
+
|
|
166
|
+
Decoding and encoding
|
|
167
|
+
=====================
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from neon import decode, decode_file, encode
|
|
171
|
+
|
|
172
|
+
decode("a: [1, 2]") # {'a': [1, 2]}
|
|
173
|
+
decode_file("phpstan.neon") # the value of a file
|
|
174
|
+
encode({"a": [1, 2]}) # '{a: [1, 2]}'
|
|
175
|
+
encode({"a": [1, 2]}, True) # 'a:\n\t- 1\n\t- 2\n\n'
|
|
176
|
+
encode({"a": [1, 2]}, lambda _, path: not path) # 'a: [1, 2]\n', one level of blocks
|
|
177
|
+
encode(config, lambda value, _: not isinstance(value, list)) # lists on one line
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
| NEON | Python |
|
|
181
|
+
|---|---|
|
|
182
|
+
| `null`, `true`, `yes` ... | `None`, `bool` |
|
|
183
|
+
| integer | `int`, of any size |
|
|
184
|
+
| float | `float` |
|
|
185
|
+
| string | `str` |
|
|
186
|
+
| date | `Date` (components and offset as written; `to_datetime()` for a `datetime`) |
|
|
187
|
+
| entity, chain | `Entity` |
|
|
188
|
+
| list | `list` |
|
|
189
|
+
| map | `dict` with string keys, in the order of the file |
|
|
190
|
+
|
|
191
|
+
The mapping is one-to-one: `1.0` stays a float, a big integer stays exact, `{}` is a `dict` and
|
|
192
|
+
`[]` a `list`, and a key keeps its place. `encode()` takes `datetime` and `date` as well.
|
|
193
|
+
`decode()` is typed as `json.loads()` is, `Any`; `to_value()` of the tree returns the exact
|
|
194
|
+
union `neon.Value`.
|
|
195
|
+
|
|
196
|
+
▶ Runnable examples: [examples/parsing](examples/parsing/), [examples/encoding](examples/encoding/)
|
|
197
|
+
|
|
198
|
+
<!---->
|
|
199
|
+
|
|
200
|
+
One format, one specification
|
|
201
|
+
=============================
|
|
202
|
+
|
|
203
|
+
The format, the value model, the tree and the semantics of editing are written down in a
|
|
204
|
+
language-neutral [specification](https://github.com/nette/neon-specification) with a conformance
|
|
205
|
+
corpus, and every implementation runs that corpus in its tests. The values and the error messages
|
|
206
|
+
were checked against nette/neon for PHP over 30,000 real-world files and 100,000 generated ones.
|
|
207
|
+
The corpus is public; if you write an implementation of NEON for another language, use it.
|
|
208
|
+
|
|
209
|
+
<!---->
|
|
210
|
+
|
|
211
|
+
For editors and AI agents
|
|
212
|
+
=========================
|
|
213
|
+
|
|
214
|
+
```shell
|
|
215
|
+
neon-lint config/
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
config/local.neon:3:2: Duplicated key 'user'
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
One line per finding, the exit code as the verdict, `--json` for tools; the same command, output
|
|
223
|
+
and exit codes as `neon-lint` of the other implementations. Without installing anything:
|
|
224
|
+
`uvx --from nette-neon neon-lint config/`.
|
|
225
|
+
|
|
226
|
+
For **AI agents**, run it after every edit of a `.neon` file and hand a finding back as an error,
|
|
227
|
+
so the agent fixes the file before it goes on. See [docs/skills/agents.md](docs/skills/agents.md).
|
|
228
|
+
|
|
229
|
+
▶ Runnable examples: [examples/linting](examples/linting/)
|
|
230
|
+
|
|
231
|
+
<!---->
|
|
232
|
+
|
|
233
|
+
Limits
|
|
234
|
+
======
|
|
235
|
+
|
|
236
|
+
- No formatter: the library keeps your formatting, it does not impose one.
|
|
237
|
+
- No schema or validation of values.
|
|
238
|
+
- No error recovery: an invalid file is reported, not partially parsed.
|
|
239
|
+
- Positions are computed on request, from the current tree.
|
|
240
|
+
|
|
241
|
+
<!---->
|
|
242
|
+
|
|
243
|
+
Documentation and credits
|
|
244
|
+
=========================
|
|
245
|
+
|
|
246
|
+
The [examples](examples/) are the documentation of the API, each chapter with the real output of
|
|
247
|
+
its programs. The format itself is documented in the [Nette manual](https://doc.nette.org/en/neon).
|
|
248
|
+
|
|
249
|
+
Released under the [MIT license](license.md).
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# Internals
|
|
2
|
+
|
|
3
|
+
How `nette-neon` works where it is not obvious from the code. The format, the tree and the
|
|
4
|
+
semantics of editing are defined by the specification (`tests/neon-spec/spec/`); the
|
|
5
|
+
implementation is a port of `@nette/neon` for JavaScript, module by module, so the two can be
|
|
6
|
+
read side by side. This document covers what is specific to this implementation.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
## Map
|
|
10
|
+
|
|
11
|
+
| module | role |
|
|
12
|
+
|---|---|
|
|
13
|
+
| `scanner.py` | the hand-written scanner of grammar.md §1, `TokenKind`, `TriviaKind` |
|
|
14
|
+
| `lexer.py` | tokens with trivia attached by the convention; `decode_utf8()`; `quote()` of messages |
|
|
15
|
+
| `token.py` | `Token`, `Trivia`, the setters that keep trivia canonical, neighbours of a token |
|
|
16
|
+
| `parser.py` | the parser of nette/neon 3.x ported onto tokens |
|
|
17
|
+
| `nodes.py` | node classes, slots, queries, the facade methods of editing |
|
|
18
|
+
| `values.py` | evaluation of literals and strings, canonical keys, `Date`, `Entity`, equality |
|
|
19
|
+
| `encoder.py` | `encode()`, the one source of how a value is written |
|
|
20
|
+
| `builder.py` | nodes from host values: encode, then parse |
|
|
21
|
+
| `editing.py` | every mutation (mutations.md) |
|
|
22
|
+
| `layout.py` | indentation unit, indentation of an item, line ending |
|
|
23
|
+
| `_journal.py` | `atomic()` and `record()`, the undo of a refused edit, per thread and task |
|
|
24
|
+
| `facade.py` | `decode()`, `decode_file()`, `parse_file()` |
|
|
25
|
+
| `printer.py`, `dumper.py`, `traverser.py`, `lint.py` | as named |
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
## Parsing
|
|
29
|
+
|
|
30
|
+
- **The parser is the algorithm of 3.x**, kept parallel to `parser.ts` and to
|
|
31
|
+
`Nette\Neon\Parser`: the error messages and positions come from the order in which it looks
|
|
32
|
+
at tokens. Line endings are trivia, but the stream presents a line break before a token as
|
|
33
|
+
the pseudo-kind `Newline`, consumed once per gap.
|
|
34
|
+
- **In an inline array, a value on the line after a key is the value of that key**, unlike in
|
|
35
|
+
3.x (specification 1.1). It is read once; when a colon follows it, it is handed over as the key
|
|
36
|
+
of the next item instead of being parsed again.
|
|
37
|
+
- **The block on the line of a dash does not backtrack**: its indentation is a `_PendingIndent`
|
|
38
|
+
decided on the line where the block first compares it.
|
|
39
|
+
- **Positions are code points, which is what a Python `str` is indexed by**, so an offset is an
|
|
40
|
+
index into the string and `Cursor` only counts lines.
|
|
41
|
+
- **Integers have any size.** `int()` and `str()` refuse more than 4300 digits, so beyond about
|
|
42
|
+
4000 digits `values.py` converts through `Decimal`, which has no such limit.
|
|
43
|
+
- **Arrays nest at most 500 levels** (`MAX_DEPTH`), and walking such a tree recursively
|
|
44
|
+
(`to_value()`, `get_tokens()`, `dump()`, the parser itself) takes several frames a level,
|
|
45
|
+
more than the default recursion limit of 1000. Importing the package raises the limit to at
|
|
46
|
+
least 10,000 frames; since Python 3.12 that costs no C stack.
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
## The tree
|
|
50
|
+
|
|
51
|
+
- Slots are private attributes behind read-only properties; a slot is written through
|
|
52
|
+
`_adopt()`, which refuses a node with a parent or one that would become its own ancestor. The
|
|
53
|
+
raw writes of the editing code are `_set_slots()`, `_set_items()`, `_set_value()`,
|
|
54
|
+
`_set_entities()`, `_set_leading()` and `_set_trailing()`; the public setters
|
|
55
|
+
(`item.value = node`, `document.value = node`, the trivia setters of a token) are editing
|
|
56
|
+
operations or check the neighbours.
|
|
57
|
+
- Lists are tuples, so `items`, `entities` and the trivia of a token can be returned without a
|
|
58
|
+
copy and cannot be changed from outside. Identity matters: `Trivia` is a frozen dataclass and
|
|
59
|
+
two trivia with the same text are equal, so the code that looks for one (`replace_trivia()`,
|
|
60
|
+
the snapshots of the tests) compares with `is`.
|
|
61
|
+
- `nodes.py` imports `editing.py` inside its methods, and `token.py` imports `nodes.py` inside
|
|
62
|
+
its functions, because each needs the other; keep the imports there.
|
|
63
|
+
- `TokenKind` is a `StrEnum` whose values are the names of the specification, so the dump
|
|
64
|
+
prints them as they are and `token.is_(":")` (a text) is told from
|
|
65
|
+
`token.is_(TokenKind.COLON)` (a kind) by the type.
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
## Editing
|
|
69
|
+
|
|
70
|
+
- **A refused edit changes nothing.** The checks that can run first do so, and every public
|
|
71
|
+
mutation runs in `with atomic():`, which collects how to undo each raw write (`record()`) and
|
|
72
|
+
replays them in reverse when the block raises. A new mutation must run in `atomic()`, and a
|
|
73
|
+
new private attribute that an edit writes must `record()` its old value.
|
|
74
|
+
- The rest follows `editing.ts` function by function: `replace_node()` and `detach_node()` decide
|
|
75
|
+
by the slot, writing an equal value is a no-op by `values_equal()`, new text comes from
|
|
76
|
+
`encode()` through `Builder`, and every operation that changes an array ends with
|
|
77
|
+
`_keep_keys()` and `_guard_grammar()`.
|
|
78
|
+
- A Python `dict` keeps its order, so a host map is compared with the tree in order, and the
|
|
79
|
+
key of a `dict` may be a `str` or an `int` (never a `bool`).
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
## Tests
|
|
83
|
+
|
|
84
|
+
- `tests/test_corpus.py` and `tests/test_mutations.py` run every category of the corpus,
|
|
85
|
+
loaded by the strict validator of `tests/corpus.py`; `tests/test_mutation_fuzz.py` edits the
|
|
86
|
+
documents of the corpus at random against an independent model.
|
|
87
|
+
- `tests/differential.py` compares with nette/neon 3.4 over the output of
|
|
88
|
+
`php-oracle.php outcomes` or `fuzz.php` from the specification repository.
|
|
89
|
+
- `examples/verify.py` runs the examples and compares their output with the readmes.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# NEON for AI agents
|
|
2
|
+
|
|
3
|
+
An agent that edits configuration files should learn about a broken file the moment it breaks
|
|
4
|
+
it, not when the application fails to start. This guide sets that up with `neon-lint`, and shows
|
|
5
|
+
how an agent should edit NEON from a Python script when it has to.
|
|
6
|
+
|
|
7
|
+
`neon-lint` has the same output and exit codes in every implementation of NEON. In a Python
|
|
8
|
+
project, use this one; in any other, the JavaScript package `@nette/neon` has a ready hook for
|
|
9
|
+
Claude Code in its guide.
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## 1. Run the linter
|
|
13
|
+
|
|
14
|
+
```shell
|
|
15
|
+
uv add --dev nette-neon
|
|
16
|
+
uv run neon-lint config/
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
or without installing anything, `uvx --from nette-neon neon-lint config/` (it needs the
|
|
20
|
+
network the first time, so a hook should call the installed one).
|
|
21
|
+
|
|
22
|
+
The output is one line per finding, `<path>:<line>:<column>: <message>`, and the exit code is the
|
|
23
|
+
verdict: 0 valid, 1 a finding, 2 a wrong invocation or a failure of the linter itself.
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
## 2. After every edit
|
|
27
|
+
|
|
28
|
+
A hook of the agent runs `neon-lint <file>` after each edit of a `.neon` file and hands the
|
|
29
|
+
output back to the agent when the exit code is 1, so it fixes the file before it goes on. Treat
|
|
30
|
+
any other non-zero code as a failure of the check, shown to the user, so that an unchecked file
|
|
31
|
+
never passes for a valid one. Where the agent has no hooks, put the rule into the instructions of
|
|
32
|
+
the project:
|
|
33
|
+
|
|
34
|
+
```markdown
|
|
35
|
+
## NEON files
|
|
36
|
+
|
|
37
|
+
- After every change of a `.neon` file, run `uv run neon-lint <file>`; exit code 0 is the only
|
|
38
|
+
acceptable result. The message says the line and the column.
|
|
39
|
+
- Indentation is structure: a tab, or the spaces the file already uses. Never mix them in one block.
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## 3. Editing NEON from a program
|
|
44
|
+
|
|
45
|
+
When an agent writes a script that changes configuration, it should edit the tree, not the text
|
|
46
|
+
with regular expressions. The tree keeps comments, blank lines, indentation and the style of
|
|
47
|
+
values, and every edit leaves a valid file:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
|
|
52
|
+
from neon import parse_file
|
|
53
|
+
|
|
54
|
+
path = Path("config/common.neon")
|
|
55
|
+
doc = parse_file(path)
|
|
56
|
+
doc.set_value(["database", "host"], "db.example.com")
|
|
57
|
+
doc.set_value(["extensions", "my"], "App\\DI\\MyExtension")
|
|
58
|
+
doc.remove_item(["parameters", "legacyOption"])
|
|
59
|
+
path.write_text(doc.print(), encoding="utf-8", newline="")
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`set_value()` on a path whose last key is missing adds it, a missing key in the middle of the path
|
|
63
|
+
is an error (a typo does not create a section), and writing a value that is already there changes
|
|
64
|
+
nothing. Write the file with `newline=""`, so that Python does not translate its line endings.
|
|
65
|
+
See `examples/editing` for what each operation does to the text.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The core of a configuration editor: list the options of a file as the fields of a form,
|
|
3
|
+
apply what the user changed, and save a file whose diff is exactly those changes.
|
|
4
|
+
|
|
5
|
+
Usage: uv run python examples/config-editor/editor.py
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from difflib import ndiff
|
|
10
|
+
from textwrap import dedent
|
|
11
|
+
|
|
12
|
+
from neon import ArrayNode, Node, PathSegment, parse
|
|
13
|
+
|
|
14
|
+
file = dedent("""\
|
|
15
|
+
# Application settings
|
|
16
|
+
# (edited by hand and by the admin panel)
|
|
17
|
+
|
|
18
|
+
mail:
|
|
19
|
+
\thost: smtp.example.com # outgoing server
|
|
20
|
+
\tport: 587
|
|
21
|
+
\tsecure: yes
|
|
22
|
+
|
|
23
|
+
cache:
|
|
24
|
+
\tenabled: true
|
|
25
|
+
\tttl: 3600 # seconds
|
|
26
|
+
\tpaths: [temp/cache]
|
|
27
|
+
""")
|
|
28
|
+
|
|
29
|
+
doc = parse(file)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# 1. the fields of the form: every scalar with its path, its type and its comment
|
|
33
|
+
def fields(node: Node | None, path: list[PathSegment]) -> Iterator[tuple]:
|
|
34
|
+
if not isinstance(node, ArrayNode):
|
|
35
|
+
return
|
|
36
|
+
for i, item in enumerate(node.items):
|
|
37
|
+
key = item.get_key_value()
|
|
38
|
+
item_path = [*path, i if key is None else key]
|
|
39
|
+
if isinstance(item.value, ArrayNode):
|
|
40
|
+
yield from fields(item.value, item_path)
|
|
41
|
+
else:
|
|
42
|
+
comment = " ".join(
|
|
43
|
+
c.get_comment_text() for c in item.get_trailing_comments()
|
|
44
|
+
)
|
|
45
|
+
yield item_path, item.get_value(), comment
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
for path, value, comment in fields(doc.value, []):
|
|
49
|
+
key = ".".join(map(str, path))
|
|
50
|
+
print(f"{key:<14} {type(value).__name__:<8} {value!s:<18} {comment}".rstrip())
|
|
51
|
+
|
|
52
|
+
# 2. what the user submitted: all fields, most of them unchanged
|
|
53
|
+
submitted = {
|
|
54
|
+
"mail.host": "smtp.example.com",
|
|
55
|
+
"mail.port": 465,
|
|
56
|
+
"mail.secure": True,
|
|
57
|
+
"cache.enabled": False,
|
|
58
|
+
"cache.ttl": 3600,
|
|
59
|
+
"cache.paths.0": "temp/cache",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
for key, value in submitted.items():
|
|
63
|
+
doc.set_value([int(s) if s.isdigit() else s for s in key.split(".")], value)
|
|
64
|
+
|
|
65
|
+
# 3. save: only what changed is in the diff
|
|
66
|
+
print()
|
|
67
|
+
for line in ndiff(file.splitlines(), doc.print().splitlines()):
|
|
68
|
+
if line[0] in "-+":
|
|
69
|
+
print(line)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# A configuration editor: a form over a NEON file
|
|
2
|
+
|
|
3
|
+
Use case number one: an admin panel shows the options of `config.neon` as the fields of a form,
|
|
4
|
+
the user changes two of them and saves, and the file keeps its comments, its blank lines and
|
|
5
|
+
everything the user did not touch.
|
|
6
|
+
|
|
7
|
+
**You will learn:**
|
|
8
|
+
|
|
9
|
+
- how to list the options of a file with their paths, types and comments
|
|
10
|
+
- why saving all fields of a form changes only those the user changed
|
|
11
|
+
|
|
12
|
+
```shell
|
|
13
|
+
uv run python examples/config-editor/editor.py
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Load, edit, save (editor.py)
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
mail.host str smtp.example.com outgoing server
|
|
20
|
+
mail.port int 587
|
|
21
|
+
mail.secure bool True
|
|
22
|
+
cache.enabled bool True
|
|
23
|
+
cache.ttl int 3600 seconds
|
|
24
|
+
cache.paths.0 str temp/cache
|
|
25
|
+
|
|
26
|
+
- port: 587
|
|
27
|
+
+ port: 465
|
|
28
|
+
- enabled: true
|
|
29
|
+
+ enabled: false
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The fields come from the tree, not from the value: the tree knows the comment that explains an
|
|
33
|
+
option and could tell the type a schema would expect. The form submits every field, and every
|
|
34
|
+
one is written with `set_value()`; four of them carry the value they already had, and writing an
|
|
35
|
+
equal value changes nothing, so the diff holds the two changes and nothing else. `secure: yes`
|
|
36
|
+
stays `yes`, though the form sent `True`.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The corners of the format that a tool meets in real files, and what the library does.
|
|
3
|
+
|
|
4
|
+
Usage: uv run python examples/edge-cases/edges.py
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
|
|
9
|
+
from neon import NeonError, decode, encode, parse
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def show(label: str, text: str) -> None:
|
|
13
|
+
try:
|
|
14
|
+
print(f"{label:<26}", json.dumps(decode(text)))
|
|
15
|
+
except NeonError as e:
|
|
16
|
+
print(f"{label:<26}", e)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
show("BOM and CRLF", "\ufeffa: 1\r\nb: 2\r\n")
|
|
20
|
+
show("dash subblock", "list:\n- a\n- b\nnext: 1")
|
|
21
|
+
show("block on a dash, tab", "- a: 1\n\tb: 2")
|
|
22
|
+
show("block on a dash, 2 spaces", "- a: 1\n b: 2")
|
|
23
|
+
show("block on a dash, 4 spaces", "- a: 1\n b: 2")
|
|
24
|
+
show("value on the next line", "a:\n\tx")
|
|
25
|
+
show("multiline indentation", "a: '''\n\t\tdeeper\n\tshallower\n'''")
|
|
26
|
+
show("words that are not bool", "[on, off, tRuE]")
|
|
27
|
+
show("implicit keys", "{5: a, b}")
|
|
28
|
+
|
|
29
|
+
# edits keep what is not edited, the missing final line ending included
|
|
30
|
+
doc = parse("a: 1\r\nb: 2")
|
|
31
|
+
doc.set_value(["c"], 3)
|
|
32
|
+
print("\nadd to CRLF without EOL ", json.dumps(doc.print()))
|
|
33
|
+
|
|
34
|
+
# a value whose first line starts with spaces is written so that it reads back the same
|
|
35
|
+
value = " indented first line\nsecond"
|
|
36
|
+
print(
|
|
37
|
+
"encode leading spaces ",
|
|
38
|
+
json.dumps(encode(value)),
|
|
39
|
+
decode(encode(value)) == value,
|
|
40
|
+
)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Edge cases: the corners real files have
|
|
2
|
+
|
|
3
|
+
NEON has a few rules that surprise a newcomer, and a tool that reads and writes configuration
|
|
4
|
+
files meets all of them sooner or later. They are the rules of nette/neon, which every Nette
|
|
5
|
+
application has read since 2011; this library keeps them, so a file means the same in PHP and here.
|
|
6
|
+
|
|
7
|
+
**You will learn:**
|
|
8
|
+
|
|
9
|
+
- how a byte order mark, CRLF and a missing final line ending are kept
|
|
10
|
+
- the dash subblock, a block opened on the line of a dash, and a value on the next line
|
|
11
|
+
- how the indentation of a multiline string is found
|
|
12
|
+
- which words are not booleans, and how implicit keys count
|
|
13
|
+
|
|
14
|
+
```shell
|
|
15
|
+
uv run python examples/edge-cases/edges.py
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## The rules (edges.py)
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
BOM and CRLF {"a": 1, "b": 2}
|
|
22
|
+
dash subblock {"list": ["a", "b"], "next": 1}
|
|
23
|
+
block on a dash, tab [{"a": 1, "b": 2}]
|
|
24
|
+
block on a dash, 2 spaces [{"a": 1, "b": 2}]
|
|
25
|
+
block on a dash, 4 spaces Bad indentation on line 2 at column 5
|
|
26
|
+
value on the next line {"a": "x"}
|
|
27
|
+
multiline indentation {"a": "deeper\n\tshallower"}
|
|
28
|
+
words that are not bool ["on", "off", "tRuE"]
|
|
29
|
+
implicit keys {"5": "a", "6": "b"}
|
|
30
|
+
|
|
31
|
+
add to CRLF without EOL "a: 1\r\nb: 2\r\nc: 3"
|
|
32
|
+
encode leading spaces "\"\"\"\n\t\\u0020\\u0020indented first line\n\tsecond\n\"\"\"" True
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
A **dash subblock** is a list written at the same indentation as its key; it ends at the first
|
|
36
|
+
item with a key. A **block opened on the line of a dash** (`- a: 1`) continues on the next line
|
|
37
|
+
indented by a tab or by two spaces, nothing else, which is why four spaces are an error. The
|
|
38
|
+
**indentation of a multiline string** is that of its first line; a line indented less keeps its
|
|
39
|
+
whitespace. `on`, `off` and odd spellings like `tRuE` are strings, though the encoder quotes them.
|
|
40
|
+
An item without a key gets **one more than the greatest integer key** before it.
|
|
41
|
+
|
|
42
|
+
Edits keep the line endings of the file and its missing final line ending. And the encoder writes
|
|
43
|
+
a string whose first line starts with spaces in `"""` with escapes, because in `'''` those spaces
|
|
44
|
+
would be read as indentation.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Adding items: a new key gets the indentation of its siblings, a list item its dash, a
|
|
3
|
+
missing section is created under the key that exists, and a typo in a path fails instead
|
|
4
|
+
of adding a key.
|
|
5
|
+
|
|
6
|
+
Usage: uv run python examples/editing/adding.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from difflib import ndiff
|
|
10
|
+
from textwrap import dedent
|
|
11
|
+
|
|
12
|
+
from neon import NeonError, parse
|
|
13
|
+
|
|
14
|
+
text = dedent("""\
|
|
15
|
+
parameters:
|
|
16
|
+
mail: info@example.com
|
|
17
|
+
|
|
18
|
+
services:
|
|
19
|
+
\t- App\\Model\\UserFacade
|
|
20
|
+
|
|
21
|
+
extensions:
|
|
22
|
+
""")
|
|
23
|
+
|
|
24
|
+
doc = parse(text)
|
|
25
|
+
doc.set_value(["parameters", "admin"], "admin@example.com")
|
|
26
|
+
doc.add_item(["services"], "App\\Model\\OrderFacade")
|
|
27
|
+
doc.set_value(["extensions", "console"], "Contributte\\Console\\DI\\ConsoleExtension")
|
|
28
|
+
|
|
29
|
+
for line in ndiff(text.splitlines(), doc.print().splitlines()):
|
|
30
|
+
if line[0] in "-+":
|
|
31
|
+
print(line)
|
|
32
|
+
|
|
33
|
+
print()
|
|
34
|
+
try:
|
|
35
|
+
doc.set_value(["parametres", "admin"], "x")
|
|
36
|
+
except NeonError as e:
|
|
37
|
+
print(e)
|
|
38
|
+
try:
|
|
39
|
+
doc.set_value(["parameters", "amdin"], "x", create=False)
|
|
40
|
+
except NeonError as e:
|
|
41
|
+
print(e)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Inline arrays: an item is added with the separator its neighbours use, on its own line in
|
|
3
|
+
an array that spans lines, and the trailing comma stays where it was.
|
|
4
|
+
|
|
5
|
+
Usage: uv run python examples/editing/inline.py
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from difflib import ndiff
|
|
9
|
+
from textwrap import dedent
|
|
10
|
+
|
|
11
|
+
from neon import parse
|
|
12
|
+
|
|
13
|
+
text = dedent("""\
|
|
14
|
+
roles: [guest, member]
|
|
15
|
+
hosts: [
|
|
16
|
+
\talpha,
|
|
17
|
+
\tbeta,
|
|
18
|
+
]
|
|
19
|
+
limits: {cpu: 2, memory: 512}
|
|
20
|
+
empty: []
|
|
21
|
+
""")
|
|
22
|
+
|
|
23
|
+
doc = parse(text)
|
|
24
|
+
doc.add_item(["roles"], "admin")
|
|
25
|
+
doc.add_item(["roles"], "root", position=0)
|
|
26
|
+
doc.add_item(["hosts"], "gamma")
|
|
27
|
+
doc.set_value(["limits", "disk"], 100)
|
|
28
|
+
doc.remove_item(["limits", "cpu"])
|
|
29
|
+
doc.add_item(["empty"], "first")
|
|
30
|
+
|
|
31
|
+
for line in ndiff(text.splitlines(), doc.print().splitlines()):
|
|
32
|
+
if line[0] in "-+":
|
|
33
|
+
print(line)
|