python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- delphi_lsp/__init__.py +184 -0
- delphi_lsp/_version.py +1 -0
- delphi_lsp/agent_cache.py +1574 -0
- delphi_lsp/agent_cli.py +902 -0
- delphi_lsp/agent_context.py +3907 -0
- delphi_lsp/agent_cpg.py +349 -0
- delphi_lsp/agent_cpg_builder.py +666 -0
- delphi_lsp/agent_layers.py +1410 -0
- delphi_lsp/agent_metrics.py +526 -0
- delphi_lsp/agent_protocol.py +445 -0
- delphi_lsp/agent_relations.py +1881 -0
- delphi_lsp/agent_templates.py +614 -0
- delphi_lsp/agent_wiki.py +2853 -0
- delphi_lsp/agent_workspace.py +860 -0
- delphi_lsp/ast_serialize.py +126 -0
- delphi_lsp/binary.py +250 -0
- delphi_lsp/comment_builder.py +28 -0
- delphi_lsp/consts.py +345 -0
- delphi_lsp/delphi_lsp_native.pyd +0 -0
- delphi_lsp/delphiast_lexer.py +399 -0
- delphi_lsp/delphiast_parser.py +2189 -0
- delphi_lsp/delphiast_tokens.py +41 -0
- delphi_lsp/grammar.py +459 -0
- delphi_lsp/incremental.py +1139 -0
- delphi_lsp/lark_builder.py +2684 -0
- delphi_lsp/lark_tokens.py +236 -0
- delphi_lsp/lexical_scanner.py +30 -0
- delphi_lsp/lsp_server.py +6698 -0
- delphi_lsp/metrics.py +1452 -0
- delphi_lsp/native_bridge.py +300 -0
- delphi_lsp/navigation_cache.py +189 -0
- delphi_lsp/nodes.py +462 -0
- delphi_lsp/parallel_outline.py +662 -0
- delphi_lsp/parse_artifacts.py +969 -0
- delphi_lsp/parser.py +342 -0
- delphi_lsp/parser_backend.py +139 -0
- delphi_lsp/preprocessor.py +1122 -0
- delphi_lsp/progress.py +26 -0
- delphi_lsp/project_config.py +285 -0
- delphi_lsp/project_discovery.py +932 -0
- delphi_lsp/project_indexer.py +470 -0
- delphi_lsp/py.typed +1 -0
- delphi_lsp/response_cache.py +465 -0
- delphi_lsp/semantic.py +394 -0
- delphi_lsp/semantic_builder.py +2302 -0
- delphi_lsp/source_reader.py +17 -0
- delphi_lsp/wiki_external_sort.py +392 -0
- delphi_lsp/wiki_memory.py +548 -0
- delphi_lsp/workspace.py +83 -0
- delphi_lsp/writer.py +73 -0
- python_delphi_lsp-3.6.0.dist-info/METADATA +766 -0
- python_delphi_lsp-3.6.0.dist-info/RECORD +56 -0
- python_delphi_lsp-3.6.0.dist-info/WHEEL +4 -0
- python_delphi_lsp-3.6.0.dist-info/entry_points.txt +3 -0
- python_delphi_lsp-3.6.0.dist-info/licenses/LICENSE +373 -0
- python_delphi_lsp-3.6.0.dist-info/sboms/python-delphi-lsp.cyclonedx.json +770 -0
|
@@ -0,0 +1,2189 @@
|
|
|
1
|
+
"""Tolerant pure-Python structural parser inspired by DelphiAST/SimpleParser.
|
|
2
|
+
|
|
3
|
+
This file is a clean Python port of the parser architecture used by DelphiAST.
|
|
4
|
+
DelphiAST is licensed under MPL-2.0:
|
|
5
|
+
https://github.com/RomanYankovsky/DelphiAST
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import gc
|
|
11
|
+
import threading
|
|
12
|
+
from collections.abc import Callable, Iterable
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from functools import wraps
|
|
15
|
+
from typing import ParamSpec, TypeVar
|
|
16
|
+
|
|
17
|
+
from .consts import AttributeName, SyntaxNodeType
|
|
18
|
+
from .delphiast_lexer import DelphiAstLexer
|
|
19
|
+
from .delphiast_tokens import Token, TokenKind
|
|
20
|
+
from .nodes import CompoundSyntaxNode, SyntaxNode, ValuedSyntaxNode
|
|
21
|
+
|
|
22
|
+
_P = ParamSpec("_P")
|
|
23
|
+
_R = TypeVar("_R")
|
|
24
|
+
_GC_STATE_LOCK = threading.Lock()
|
|
25
|
+
_gc_suspensions = 0
|
|
26
|
+
_restore_gc = False
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _begin_gc_suspension() -> None:
|
|
30
|
+
global _gc_suspensions, _restore_gc
|
|
31
|
+
with _GC_STATE_LOCK:
|
|
32
|
+
if _gc_suspensions == 0:
|
|
33
|
+
_restore_gc = gc.isenabled()
|
|
34
|
+
if _restore_gc:
|
|
35
|
+
gc.disable()
|
|
36
|
+
_gc_suspensions += 1
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _end_gc_suspension() -> None:
|
|
40
|
+
global _gc_suspensions, _restore_gc
|
|
41
|
+
with _GC_STATE_LOCK:
|
|
42
|
+
_gc_suspensions -= 1
|
|
43
|
+
if _gc_suspensions == 0:
|
|
44
|
+
if _restore_gc:
|
|
45
|
+
gc.enable()
|
|
46
|
+
_restore_gc = False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _without_cyclic_gc(function: Callable[_P, _R]) -> Callable[_P, _R]:
|
|
50
|
+
@wraps(function)
|
|
51
|
+
def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
|
52
|
+
_begin_gc_suspension()
|
|
53
|
+
try:
|
|
54
|
+
return function(*args, **kwargs)
|
|
55
|
+
finally:
|
|
56
|
+
_end_gc_suspension()
|
|
57
|
+
|
|
58
|
+
return wrapped
|
|
59
|
+
|
|
60
|
+
_ROUTINES = frozenset({"procedure", "function", "constructor", "destructor", "operator"})
|
|
61
|
+
_SECTIONS = frozenset(
|
|
62
|
+
{
|
|
63
|
+
"interface",
|
|
64
|
+
"implementation",
|
|
65
|
+
"initialization",
|
|
66
|
+
"finalization",
|
|
67
|
+
"type",
|
|
68
|
+
"const",
|
|
69
|
+
"resourcestring",
|
|
70
|
+
"var",
|
|
71
|
+
"threadvar",
|
|
72
|
+
"label",
|
|
73
|
+
"exports",
|
|
74
|
+
"uses",
|
|
75
|
+
"contains",
|
|
76
|
+
"requires",
|
|
77
|
+
"begin",
|
|
78
|
+
"asm",
|
|
79
|
+
"end",
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
_VISIBILITIES = {
|
|
83
|
+
"private": SyntaxNodeType.ntPrivate,
|
|
84
|
+
"protected": SyntaxNodeType.ntProtected,
|
|
85
|
+
"public": SyntaxNodeType.ntPublic,
|
|
86
|
+
"published": SyntaxNodeType.ntPublished,
|
|
87
|
+
}
|
|
88
|
+
_METHOD_DIRECTIVES = {
|
|
89
|
+
"abstract": AttributeName.anAbstract,
|
|
90
|
+
"assembler": AttributeName.anAssembler,
|
|
91
|
+
"deprecated": AttributeName.anDeprecated,
|
|
92
|
+
"external": AttributeName.anExternal,
|
|
93
|
+
"final": AttributeName.anFinal,
|
|
94
|
+
"forward": AttributeName.anForwarded,
|
|
95
|
+
"inline": AttributeName.anInline,
|
|
96
|
+
"overload": AttributeName.anOverload,
|
|
97
|
+
"reintroduce": AttributeName.anReintroduce,
|
|
98
|
+
"sealed": AttributeName.anSealed,
|
|
99
|
+
"static": AttributeName.anStatic,
|
|
100
|
+
"unsafe": AttributeName.anUnsafe,
|
|
101
|
+
"noreturn": AttributeName.anNoReturn,
|
|
102
|
+
}
|
|
103
|
+
_CALLING_CONVENTIONS = frozenset(
|
|
104
|
+
{
|
|
105
|
+
"cdecl",
|
|
106
|
+
"msfastcall",
|
|
107
|
+
"pascal",
|
|
108
|
+
"register",
|
|
109
|
+
"safecall",
|
|
110
|
+
"softfloat",
|
|
111
|
+
"stdcall",
|
|
112
|
+
"syscall",
|
|
113
|
+
"vectorcall",
|
|
114
|
+
"winapi",
|
|
115
|
+
}
|
|
116
|
+
)
|
|
117
|
+
_DECLARATION_MODIFIERS = frozenset(
|
|
118
|
+
{
|
|
119
|
+
"cvar",
|
|
120
|
+
"deprecated",
|
|
121
|
+
"experimental",
|
|
122
|
+
"export",
|
|
123
|
+
"external",
|
|
124
|
+
"far",
|
|
125
|
+
"library",
|
|
126
|
+
"near",
|
|
127
|
+
"platform",
|
|
128
|
+
"public",
|
|
129
|
+
"weakexternal",
|
|
130
|
+
"extdecl",
|
|
131
|
+
"varargs",
|
|
132
|
+
}
|
|
133
|
+
) | _CALLING_CONVENTIONS
|
|
134
|
+
_STATEMENT_TERMINATORS = frozenset({"end", "else", "until", "except", "finally"})
|
|
135
|
+
_PROPERTY_ACCESSORS = frozenset(
|
|
136
|
+
{"read", "write", "readonly", "writeonly", "add", "remove"}
|
|
137
|
+
)
|
|
138
|
+
_PROPERTY_STORAGE = frozenset({"default", "nodefault", "stored"})
|
|
139
|
+
_PROPERTY_SPECIFIERS = frozenset(
|
|
140
|
+
{
|
|
141
|
+
"index",
|
|
142
|
+
"dispid",
|
|
143
|
+
"implements",
|
|
144
|
+
*_PROPERTY_ACCESSORS,
|
|
145
|
+
*_PROPERTY_STORAGE,
|
|
146
|
+
}
|
|
147
|
+
)
|
|
148
|
+
_DEFERRED_FLAT_BLOCK_TOKENS = 512
|
|
149
|
+
_DEFERRED_BLOCK_OPENERS = frozenset(
|
|
150
|
+
{
|
|
151
|
+
"asm",
|
|
152
|
+
"begin",
|
|
153
|
+
"case",
|
|
154
|
+
"class",
|
|
155
|
+
"dispinterface",
|
|
156
|
+
"interface",
|
|
157
|
+
"object",
|
|
158
|
+
"record",
|
|
159
|
+
"try",
|
|
160
|
+
}
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True)
|
|
165
|
+
class DelphiAstParseProblem:
|
|
166
|
+
message: str
|
|
167
|
+
line: int
|
|
168
|
+
column: int
|
|
169
|
+
category: str = "syntax"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@dataclass
|
|
173
|
+
class DelphiAstParseOutput:
|
|
174
|
+
root: SyntaxNode
|
|
175
|
+
problems: list[DelphiAstParseProblem]
|
|
176
|
+
requires_legacy: bool = False
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class _NodePositionAnchor:
|
|
180
|
+
__slots__ = (
|
|
181
|
+
"_positions",
|
|
182
|
+
"column",
|
|
183
|
+
"end_column",
|
|
184
|
+
"end_line",
|
|
185
|
+
"line",
|
|
186
|
+
"value_length",
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
def __init__(
|
|
190
|
+
self,
|
|
191
|
+
line: int,
|
|
192
|
+
column: int,
|
|
193
|
+
end_line: int,
|
|
194
|
+
end_column: int,
|
|
195
|
+
value_length: int,
|
|
196
|
+
) -> None:
|
|
197
|
+
self.line = line
|
|
198
|
+
self.column = column
|
|
199
|
+
self.end_line = end_line
|
|
200
|
+
self.end_column = end_column
|
|
201
|
+
self.value_length = value_length
|
|
202
|
+
self._positions: tuple[tuple[int, int], ...] | None = None
|
|
203
|
+
|
|
204
|
+
@property
|
|
205
|
+
def positions(self) -> tuple[tuple[int, int], ...]:
|
|
206
|
+
positions = self._positions
|
|
207
|
+
if positions is None:
|
|
208
|
+
positions = tuple(
|
|
209
|
+
(self.line, column)
|
|
210
|
+
for column in range(
|
|
211
|
+
self.column,
|
|
212
|
+
self.column + self.value_length + 1,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
self._positions = positions
|
|
216
|
+
return positions
|
|
217
|
+
|
|
218
|
+
@property
|
|
219
|
+
def positions_materialized(self) -> bool:
|
|
220
|
+
return self._positions is not None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class DelphiAstParser:
|
|
224
|
+
"""Single-pass, recovery-oriented Object Pascal parser.
|
|
225
|
+
|
|
226
|
+
The parser deliberately extracts the structures consumed by indexing,
|
|
227
|
+
navigation, relations, and metrics. It does not reject an entire unit when
|
|
228
|
+
one dialect-specific declaration is unsupported.
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
@_without_cyclic_gc
|
|
232
|
+
def __init__(
|
|
233
|
+
self,
|
|
234
|
+
source: str,
|
|
235
|
+
file_name: str = "",
|
|
236
|
+
*,
|
|
237
|
+
build_expressions: bool = True,
|
|
238
|
+
build_node_positions: bool = True,
|
|
239
|
+
) -> None:
|
|
240
|
+
self.source = source
|
|
241
|
+
self.file_name = file_name
|
|
242
|
+
self.build_expressions = build_expressions
|
|
243
|
+
self.build_node_positions = build_node_positions
|
|
244
|
+
self.tokens = DelphiAstLexer(source, file_name).tokenize(include_trivia=False)
|
|
245
|
+
self._last_token_index = len(self.tokens) - 1
|
|
246
|
+
self.index = 0
|
|
247
|
+
self.problems: list[DelphiAstParseProblem] = []
|
|
248
|
+
self._node_position_cache: dict[int, _NodePositionAnchor] = {}
|
|
249
|
+
|
|
250
|
+
@_without_cyclic_gc
|
|
251
|
+
def parse(self) -> DelphiAstParseOutput:
|
|
252
|
+
root = self._node(SyntaxNodeType.ntUnit, self._peek(), compound=True)
|
|
253
|
+
header = self._normalized()
|
|
254
|
+
if header in {"unit", "program", "library", "package"}:
|
|
255
|
+
root.set_attribute(AttributeName.anKind, header)
|
|
256
|
+
self._advance()
|
|
257
|
+
name = self._read_qualified_name(allow_keywords=True)
|
|
258
|
+
if name:
|
|
259
|
+
root.set_attribute(AttributeName.anName, name)
|
|
260
|
+
else:
|
|
261
|
+
self._problem("Expected module name")
|
|
262
|
+
while not self._eof() and not self._at(";"):
|
|
263
|
+
if self._accept("deprecated"):
|
|
264
|
+
value = "true"
|
|
265
|
+
if self._peek().kind is TokenKind.STRING:
|
|
266
|
+
value = self._dequote_string(self._advance().value)
|
|
267
|
+
root.set_attribute(AttributeName.anDeprecated, value)
|
|
268
|
+
elif self._accept("experimental"):
|
|
269
|
+
root.set_attribute(AttributeName.anKind, "experimental")
|
|
270
|
+
else:
|
|
271
|
+
self._advance()
|
|
272
|
+
self._consume_through(";")
|
|
273
|
+
else:
|
|
274
|
+
self._problem("Expected unit, program, library, or package header")
|
|
275
|
+
|
|
276
|
+
current = root
|
|
277
|
+
while not self._eof():
|
|
278
|
+
before = self.index
|
|
279
|
+
word = self._normalized()
|
|
280
|
+
if word == "interface":
|
|
281
|
+
current = self._section(root, SyntaxNodeType.ntInterface)
|
|
282
|
+
elif word == "implementation":
|
|
283
|
+
current = self._section(root, SyntaxNodeType.ntImplementation)
|
|
284
|
+
elif word == "initialization":
|
|
285
|
+
current = self._section(root, SyntaxNodeType.ntInitialization)
|
|
286
|
+
self._parse_statements(current, {"finalization", "end"})
|
|
287
|
+
elif word == "finalization":
|
|
288
|
+
current = self._section(root, SyntaxNodeType.ntFinalization)
|
|
289
|
+
self._parse_statements(current, {"end"})
|
|
290
|
+
elif word in {"uses", "contains", "requires"}:
|
|
291
|
+
current.add_child(self._parse_dependencies(word))
|
|
292
|
+
elif word == "type":
|
|
293
|
+
current.add_child(self._parse_type_section())
|
|
294
|
+
elif word in {"const", "resourcestring"}:
|
|
295
|
+
current.add_child(self._parse_constants(word))
|
|
296
|
+
elif word in {"var", "threadvar"}:
|
|
297
|
+
current.add_child(self._parse_variables(word))
|
|
298
|
+
elif word == "exports":
|
|
299
|
+
current.add_child(self._parse_exports())
|
|
300
|
+
elif word in _ROUTINES or (word == "class" and self._normalized(1) in _ROUTINES):
|
|
301
|
+
current.add_child(
|
|
302
|
+
self._parse_routine(
|
|
303
|
+
parse_body=current.typ is not SyntaxNodeType.ntInterface,
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
elif word == "begin":
|
|
307
|
+
statements = self._parse_begin_block()
|
|
308
|
+
current.add_child(statements)
|
|
309
|
+
elif word == "end":
|
|
310
|
+
self._advance()
|
|
311
|
+
self._accept(".")
|
|
312
|
+
self._accept(";")
|
|
313
|
+
elif self._peek().kind is TokenKind.UNKNOWN:
|
|
314
|
+
self._problem(f"Unsupported character {self._peek().value!r}")
|
|
315
|
+
self._advance()
|
|
316
|
+
else:
|
|
317
|
+
self._advance()
|
|
318
|
+
self._ensure_progress(before)
|
|
319
|
+
|
|
320
|
+
if isinstance(root, CompoundSyntaxNode):
|
|
321
|
+
last = self.tokens[max(0, self._last_token_index)]
|
|
322
|
+
root.end_line = last.line
|
|
323
|
+
root.end_col = last.column
|
|
324
|
+
return DelphiAstParseOutput(root=root, problems=self.problems)
|
|
325
|
+
|
|
326
|
+
def _section(self, root: SyntaxNode, typ: SyntaxNodeType) -> SyntaxNode:
|
|
327
|
+
token = self._advance()
|
|
328
|
+
section = self._node(typ, token, compound=True)
|
|
329
|
+
root.add_child(section)
|
|
330
|
+
return section
|
|
331
|
+
|
|
332
|
+
def _parse_dependencies(self, keyword: str) -> SyntaxNode:
|
|
333
|
+
token = self._advance()
|
|
334
|
+
typ = {
|
|
335
|
+
"uses": SyntaxNodeType.ntUses,
|
|
336
|
+
"contains": SyntaxNodeType.ntContains,
|
|
337
|
+
"requires": SyntaxNodeType.ntRequires,
|
|
338
|
+
}[keyword]
|
|
339
|
+
section = self._node(typ, token)
|
|
340
|
+
while not self._eof() and not self._at(";"):
|
|
341
|
+
before = self.index
|
|
342
|
+
name_token = self._peek()
|
|
343
|
+
name = self._read_qualified_name(allow_keywords=True)
|
|
344
|
+
if name:
|
|
345
|
+
unit = self._node(SyntaxNodeType.ntUnit, name_token)
|
|
346
|
+
unit.set_attribute(AttributeName.anName, name)
|
|
347
|
+
if self._accept("in"):
|
|
348
|
+
path = self._advance()
|
|
349
|
+
unit.set_attribute(AttributeName.anPath, path.value.strip("'"))
|
|
350
|
+
section.add_child(unit)
|
|
351
|
+
else:
|
|
352
|
+
self._advance()
|
|
353
|
+
if not self._accept(",") and not self._at(";") and name:
|
|
354
|
+
self._problem("Expected ',' or ';' in dependency clause")
|
|
355
|
+
self._ensure_progress(before)
|
|
356
|
+
self._accept(";")
|
|
357
|
+
return section
|
|
358
|
+
|
|
359
|
+
def _parse_exports(self) -> SyntaxNode:
|
|
360
|
+
section = self._node(SyntaxNodeType.ntExports, self._advance())
|
|
361
|
+
while not self._eof() and not self._at(";"):
|
|
362
|
+
before = self.index
|
|
363
|
+
token = self._peek()
|
|
364
|
+
name = self._read_qualified_name(allow_keywords=True)
|
|
365
|
+
if name:
|
|
366
|
+
element = self._node(SyntaxNodeType.ntElement, token)
|
|
367
|
+
name_node = self._node(SyntaxNodeType.ntName, token)
|
|
368
|
+
name_node.set_attribute(AttributeName.anName, name)
|
|
369
|
+
element.add_child(name_node)
|
|
370
|
+
if self._normalized() in {"name", "index", "resident"}:
|
|
371
|
+
self._skip_balanced_until({",", ";"})
|
|
372
|
+
section.add_child(element)
|
|
373
|
+
else:
|
|
374
|
+
self._advance()
|
|
375
|
+
self._accept(",")
|
|
376
|
+
self._ensure_progress(before)
|
|
377
|
+
self._accept(";")
|
|
378
|
+
return section
|
|
379
|
+
|
|
380
|
+
def _parse_type_section(self, *, nested: bool = False) -> SyntaxNode:
|
|
381
|
+
section = self._node(SyntaxNodeType.ntTypeSection, self._advance())
|
|
382
|
+
while not self._eof():
|
|
383
|
+
before = self.index
|
|
384
|
+
if self._peek().kind is TokenKind.KEYWORD and (
|
|
385
|
+
self._normalized() in _SECTIONS
|
|
386
|
+
or self._normalized() in _ROUTINES
|
|
387
|
+
or (
|
|
388
|
+
self._normalized() == "class"
|
|
389
|
+
and self._normalized(1) in _ROUTINES
|
|
390
|
+
)
|
|
391
|
+
):
|
|
392
|
+
break
|
|
393
|
+
if nested and (
|
|
394
|
+
self._normalized() in _VISIBILITIES
|
|
395
|
+
or self._normalized() == "strict"
|
|
396
|
+
or self._normalized() == "property"
|
|
397
|
+
or (
|
|
398
|
+
self._normalized() == "class"
|
|
399
|
+
and self._normalized(1) in _ROUTINES | {"property"}
|
|
400
|
+
)
|
|
401
|
+
):
|
|
402
|
+
break
|
|
403
|
+
attributes = self._parse_attributes() if self._at("[") else None
|
|
404
|
+
if not self._is_name(allow_keywords=True):
|
|
405
|
+
self._problem("Expected type declaration")
|
|
406
|
+
self._synchronize({";"})
|
|
407
|
+
self._accept(";")
|
|
408
|
+
self._ensure_progress(before)
|
|
409
|
+
continue
|
|
410
|
+
generic = self._accept("generic")
|
|
411
|
+
name_token = self._peek()
|
|
412
|
+
name = self._read_qualified_name(allow_keywords=True)
|
|
413
|
+
declaration = self._node(SyntaxNodeType.ntTypeDecl, name_token)
|
|
414
|
+
declaration.set_attribute(AttributeName.anName, name)
|
|
415
|
+
if attributes is not None:
|
|
416
|
+
declaration.add_child(attributes)
|
|
417
|
+
if generic:
|
|
418
|
+
declaration.set_attribute(AttributeName.anKind, "generic")
|
|
419
|
+
self._parse_type_parameters(declaration)
|
|
420
|
+
if not self._accept("="):
|
|
421
|
+
if nested:
|
|
422
|
+
self.index = before
|
|
423
|
+
break
|
|
424
|
+
self._problem(f"Expected '=' after type {name}")
|
|
425
|
+
self._synchronize({";"})
|
|
426
|
+
self._accept(";")
|
|
427
|
+
section.add_child(declaration)
|
|
428
|
+
self._ensure_progress(before)
|
|
429
|
+
continue
|
|
430
|
+
type_node = self._parse_type()
|
|
431
|
+
if self._accept("align"):
|
|
432
|
+
type_node.set_attribute(AttributeName.anAlign, "align")
|
|
433
|
+
if self._peek().kind is TokenKind.NUMBER:
|
|
434
|
+
self._advance()
|
|
435
|
+
while self._normalized() in {
|
|
436
|
+
"deprecated",
|
|
437
|
+
"experimental",
|
|
438
|
+
"platform",
|
|
439
|
+
"library",
|
|
440
|
+
}:
|
|
441
|
+
directive = self._advance().normalized
|
|
442
|
+
if directive == "deprecated":
|
|
443
|
+
value = "true"
|
|
444
|
+
if self._peek().kind is TokenKind.STRING:
|
|
445
|
+
value = self._dequote_string(self._advance().value)
|
|
446
|
+
declaration.set_attribute(AttributeName.anDeprecated, value)
|
|
447
|
+
type_node.set_attribute(AttributeName.anDeprecated, value)
|
|
448
|
+
else:
|
|
449
|
+
declaration.set_attribute(AttributeName.anKind, directive)
|
|
450
|
+
declaration.add_child(type_node)
|
|
451
|
+
section.add_child(declaration)
|
|
452
|
+
self._accept(";")
|
|
453
|
+
while self._normalized() in {
|
|
454
|
+
"cblock",
|
|
455
|
+
"extdecl",
|
|
456
|
+
"far",
|
|
457
|
+
"near",
|
|
458
|
+
"varargs",
|
|
459
|
+
*_CALLING_CONVENTIONS,
|
|
460
|
+
}:
|
|
461
|
+
suffix = self._advance().normalized
|
|
462
|
+
if suffix == "far":
|
|
463
|
+
type_node.set_attribute(AttributeName.anFar, "true")
|
|
464
|
+
elif suffix == "near":
|
|
465
|
+
type_node.set_attribute(AttributeName.anNear, "true")
|
|
466
|
+
elif suffix == "varargs":
|
|
467
|
+
type_node.set_attribute(AttributeName.anVarArgs, "true")
|
|
468
|
+
else:
|
|
469
|
+
type_node.set_attribute(
|
|
470
|
+
AttributeName.anCallingConvention,
|
|
471
|
+
suffix,
|
|
472
|
+
)
|
|
473
|
+
self._accept(";")
|
|
474
|
+
self._ensure_progress(before)
|
|
475
|
+
return section
|
|
476
|
+
|
|
477
|
+
def _parse_type_parameters(self, declaration: SyntaxNode) -> None:
|
|
478
|
+
if not self._accept("<"):
|
|
479
|
+
return
|
|
480
|
+
params = self._node(SyntaxNodeType.ntTypeParams, self._peek(-1))
|
|
481
|
+
while not self._eof() and not self._at_type_parameter_close():
|
|
482
|
+
before = self.index
|
|
483
|
+
group = self._collect_until({";", ">", ">="})
|
|
484
|
+
colon = self._top_level_token_index(group, ":")
|
|
485
|
+
name_tokens = group if colon is None else group[:colon]
|
|
486
|
+
constraint_tokens = [] if colon is None else group[colon + 1 :]
|
|
487
|
+
names = self._split_top_level_tokens(name_tokens, {","})
|
|
488
|
+
constraints = self._parse_constraint_tokens(constraint_tokens)
|
|
489
|
+
for name_part in names:
|
|
490
|
+
if not name_part:
|
|
491
|
+
continue
|
|
492
|
+
name_token = name_part[0]
|
|
493
|
+
parameter = self._node(SyntaxNodeType.ntTypeParam, name_token)
|
|
494
|
+
parameter.add_child(
|
|
495
|
+
self._valued(
|
|
496
|
+
SyntaxNodeType.ntName,
|
|
497
|
+
name_token,
|
|
498
|
+
self._tokens_text(name_part).strip(),
|
|
499
|
+
)
|
|
500
|
+
)
|
|
501
|
+
if constraints is not None:
|
|
502
|
+
parameter.add_child(constraints.clone())
|
|
503
|
+
params.add_child(parameter)
|
|
504
|
+
self._accept(";")
|
|
505
|
+
self._ensure_progress(before)
|
|
506
|
+
self._consume_type_parameter_close()
|
|
507
|
+
declaration.add_child(params)
|
|
508
|
+
|
|
509
|
+
def _at_type_parameter_close(self) -> bool:
|
|
510
|
+
return self._at(">") or self._at(">=")
|
|
511
|
+
|
|
512
|
+
def _consume_type_parameter_close(self) -> None:
|
|
513
|
+
if self._accept(">") or not self._at(">="):
|
|
514
|
+
return
|
|
515
|
+
compound = self._peek()
|
|
516
|
+
self.tokens[self.index] = Token(
|
|
517
|
+
TokenKind.SYMBOL,
|
|
518
|
+
"=",
|
|
519
|
+
compound.line,
|
|
520
|
+
compound.column + 1,
|
|
521
|
+
compound.start + 1,
|
|
522
|
+
compound.end,
|
|
523
|
+
compound.file_name,
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
def _parse_constraint_tokens(self, values: list[Token]) -> SyntaxNode | None:
|
|
527
|
+
parts = self._split_top_level_tokens(values, {","})
|
|
528
|
+
if not any(parts):
|
|
529
|
+
return None
|
|
530
|
+
constraints = self._node(SyntaxNodeType.ntConstraints, values[0])
|
|
531
|
+
special = {
|
|
532
|
+
"class": SyntaxNodeType.ntClassConstraint,
|
|
533
|
+
"record": SyntaxNodeType.ntRecordConstraint,
|
|
534
|
+
"constructor": SyntaxNodeType.ntConstructorConstraint,
|
|
535
|
+
"interface": SyntaxNodeType.ntInterfaceConstraint,
|
|
536
|
+
"unmanaged": SyntaxNodeType.ntUnmanagedConstraint,
|
|
537
|
+
}
|
|
538
|
+
for part in parts:
|
|
539
|
+
if not part:
|
|
540
|
+
continue
|
|
541
|
+
text = self._tokens_text(part).strip()
|
|
542
|
+
typ = special.get(text.casefold())
|
|
543
|
+
if typ is not None:
|
|
544
|
+
constraints.add_child(self._node(typ, part[0]))
|
|
545
|
+
continue
|
|
546
|
+
constraint = self._node(SyntaxNodeType.ntType, part[0])
|
|
547
|
+
constraint.set_attribute(AttributeName.anName, text)
|
|
548
|
+
constraints.add_child(constraint)
|
|
549
|
+
return constraints
|
|
550
|
+
|
|
551
|
+
def _parse_type(self) -> SyntaxNode:
|
|
552
|
+
token = self._peek()
|
|
553
|
+
keyword = not self._source_token_value(token).startswith('&')
|
|
554
|
+
packing = (
|
|
555
|
+
self._advance().normalized
|
|
556
|
+
if keyword and self._normalized() in {"packed", "bitpacked"}
|
|
557
|
+
else ""
|
|
558
|
+
)
|
|
559
|
+
keyword = not self._source_token_value(self._peek()).startswith('&')
|
|
560
|
+
kind = self._normalized()
|
|
561
|
+
structured_types = {
|
|
562
|
+
"class",
|
|
563
|
+
"record",
|
|
564
|
+
"object",
|
|
565
|
+
"interface",
|
|
566
|
+
"dispinterface",
|
|
567
|
+
}
|
|
568
|
+
objective_types = {"objcclass", "objcprotocol", "objccategory"}
|
|
569
|
+
if keyword and kind == "type" and self._normalized(1) == "helper":
|
|
570
|
+
self._advance()
|
|
571
|
+
self._advance()
|
|
572
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
573
|
+
type_node.set_attribute(AttributeName.anType, "helper")
|
|
574
|
+
self._accept("for")
|
|
575
|
+
target_token = self._peek()
|
|
576
|
+
target_name = self._read_qualified_name(allow_keywords=True)
|
|
577
|
+
target = self._node(SyntaxNodeType.ntType, target_token)
|
|
578
|
+
target.set_attribute(AttributeName.anName, target_name)
|
|
579
|
+
type_node.add_child(target)
|
|
580
|
+
self._parse_type_body(type_node)
|
|
581
|
+
elif keyword and kind in structured_types | objective_types:
|
|
582
|
+
self._advance()
|
|
583
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
584
|
+
type_node.set_attribute(AttributeName.anType, "record" if kind == "object" else kind)
|
|
585
|
+
if kind in objective_types:
|
|
586
|
+
if self._accept("external"):
|
|
587
|
+
type_node.set_attribute(AttributeName.anExternal, "true")
|
|
588
|
+
if self._accept("name"):
|
|
589
|
+
if self._peek().kind is TokenKind.STRING:
|
|
590
|
+
self._advance()
|
|
591
|
+
else:
|
|
592
|
+
self._read_qualified_name(allow_keywords=True)
|
|
593
|
+
self._parse_base_types(type_node)
|
|
594
|
+
if not self._at(";"):
|
|
595
|
+
self._parse_type_body(type_node)
|
|
596
|
+
elif kind == "class" and self._accept("of"):
|
|
597
|
+
type_node.set_attribute(AttributeName.anType, "classref")
|
|
598
|
+
type_node.add_child(self._parse_type_reference({";"}))
|
|
599
|
+
else:
|
|
600
|
+
if kind in {"class", "record"} and self._accept("helper"):
|
|
601
|
+
self._accept("for")
|
|
602
|
+
target_token = self._peek()
|
|
603
|
+
target_name = self._read_qualified_name(allow_keywords=True)
|
|
604
|
+
target = self._node(SyntaxNodeType.ntType, target_token)
|
|
605
|
+
target.set_attribute(AttributeName.anName, target_name)
|
|
606
|
+
type_node.add_child(target)
|
|
607
|
+
if self._accept("abstract"):
|
|
608
|
+
type_node.set_attribute(AttributeName.anAbstract, "true")
|
|
609
|
+
self._parse_base_types(type_node)
|
|
610
|
+
if not self._at(";"):
|
|
611
|
+
self._parse_type_body(type_node)
|
|
612
|
+
elif self._at("("):
|
|
613
|
+
type_node = self._parse_enum()
|
|
614
|
+
elif keyword and kind == "set":
|
|
615
|
+
self._advance()
|
|
616
|
+
self._accept("of")
|
|
617
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
618
|
+
type_node.set_attribute(AttributeName.anType, "set")
|
|
619
|
+
type_node.add_child(self._parse_type_reference({";", ")"}))
|
|
620
|
+
elif keyword and kind == "array":
|
|
621
|
+
type_node = self._parse_array_type({";"})
|
|
622
|
+
elif keyword and kind in _ROUTINES:
|
|
623
|
+
type_node = self._parse_procedural_type()
|
|
624
|
+
elif (
|
|
625
|
+
keyword
|
|
626
|
+
and kind == "reference"
|
|
627
|
+
and self._normalized(1) == "to"
|
|
628
|
+
and self._normalized(2) in _ROUTINES
|
|
629
|
+
):
|
|
630
|
+
self._advance()
|
|
631
|
+
self._advance()
|
|
632
|
+
type_node = self._parse_procedural_type()
|
|
633
|
+
type_node.set_attribute(AttributeName.anKind, "reference")
|
|
634
|
+
wrapper = self._node(SyntaxNodeType.ntType, token)
|
|
635
|
+
wrapper.set_attribute(AttributeName.anType, "reference")
|
|
636
|
+
wrapper.add_child(type_node)
|
|
637
|
+
type_node = wrapper
|
|
638
|
+
elif self._at("^"):
|
|
639
|
+
self._advance()
|
|
640
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
641
|
+
type_node.set_attribute(AttributeName.anType, "pointer")
|
|
642
|
+
type_node.add_child(self._parse_type_reference({";", ")"}))
|
|
643
|
+
elif keyword and kind == "file":
|
|
644
|
+
self._advance()
|
|
645
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
646
|
+
type_node.set_attribute(AttributeName.anType, "file")
|
|
647
|
+
if self._accept("of"):
|
|
648
|
+
type_node.add_child(self._parse_type_annotation({";", ")"}))
|
|
649
|
+
else:
|
|
650
|
+
type_node = self._parse_type_reference({";"})
|
|
651
|
+
if packing:
|
|
652
|
+
wrapper = self._node(SyntaxNodeType.ntType, token)
|
|
653
|
+
wrapper.set_attribute(AttributeName.anType, packing)
|
|
654
|
+
wrapper.add_child(type_node)
|
|
655
|
+
return wrapper
|
|
656
|
+
return type_node
|
|
657
|
+
|
|
658
|
+
def _parse_base_types(self, type_node: SyntaxNode) -> None:
|
|
659
|
+
if not self._accept("("):
|
|
660
|
+
return
|
|
661
|
+
while not self._eof() and not self._at(")"):
|
|
662
|
+
type_node.add_child(self._parse_type_reference({",", ")"}))
|
|
663
|
+
if not self._accept(","):
|
|
664
|
+
break
|
|
665
|
+
self._accept(")")
|
|
666
|
+
|
|
667
|
+
def _parse_type_body(self, type_node: SyntaxNode) -> None:
|
|
668
|
+
active_parent = type_node
|
|
669
|
+
class_fields = False
|
|
670
|
+
while not self._eof() and not self._at("end"):
|
|
671
|
+
before = self.index
|
|
672
|
+
word = self._normalized()
|
|
673
|
+
if word in {"of", "helper"}:
|
|
674
|
+
break
|
|
675
|
+
if word == "strict" and self._normalized(1) in {"private", "protected"}:
|
|
676
|
+
first = self._advance()
|
|
677
|
+
visibility = self._advance().normalized
|
|
678
|
+
typ = (
|
|
679
|
+
SyntaxNodeType.ntStrictPrivate
|
|
680
|
+
if visibility == "private"
|
|
681
|
+
else SyntaxNodeType.ntStrictProtected
|
|
682
|
+
)
|
|
683
|
+
active_parent = self._node(typ, first)
|
|
684
|
+
type_node.add_child(active_parent)
|
|
685
|
+
class_fields = False
|
|
686
|
+
elif word in _VISIBILITIES:
|
|
687
|
+
active_parent = self._node(_VISIBILITIES[word], self._advance())
|
|
688
|
+
type_node.add_child(active_parent)
|
|
689
|
+
class_fields = False
|
|
690
|
+
elif word == "type" and self._peek().kind is TokenKind.KEYWORD:
|
|
691
|
+
active_parent.add_child(self._parse_type_section(nested=True))
|
|
692
|
+
class_fields = False
|
|
693
|
+
elif word == "const" and self._peek().kind is TokenKind.KEYWORD:
|
|
694
|
+
active_parent.add_child(self._parse_constants(word, nested=True))
|
|
695
|
+
class_fields = False
|
|
696
|
+
elif word == "class" and self._normalized(1) == "var":
|
|
697
|
+
self._advance()
|
|
698
|
+
self._advance()
|
|
699
|
+
class_fields = True
|
|
700
|
+
elif word == "var" and self._peek().kind is TokenKind.KEYWORD:
|
|
701
|
+
self._advance()
|
|
702
|
+
class_fields = False
|
|
703
|
+
elif word in {"required", "optional"}:
|
|
704
|
+
self._advance()
|
|
705
|
+
elif word == "case" and self._peek().kind is TokenKind.KEYWORD:
|
|
706
|
+
self._parse_variant_part(type_node)
|
|
707
|
+
elif word in _ROUTINES or (word == "class" and self._normalized(1) in _ROUTINES):
|
|
708
|
+
active_parent.add_child(self._parse_routine(parse_body=False))
|
|
709
|
+
elif word == "property":
|
|
710
|
+
active_parent.add_child(self._parse_property())
|
|
711
|
+
elif word == "class" and self._normalized(1) == "property":
|
|
712
|
+
self._advance()
|
|
713
|
+
prop = self._parse_property()
|
|
714
|
+
prop.set_attribute(AttributeName.anClass, "true")
|
|
715
|
+
active_parent.add_child(prop)
|
|
716
|
+
elif self._at("[") and self._peek(1).kind is TokenKind.STRING:
|
|
717
|
+
start = self._advance()
|
|
718
|
+
guid_token = self._advance()
|
|
719
|
+
guid = self._node(SyntaxNodeType.ntGuid, start)
|
|
720
|
+
guid.set_attribute(
|
|
721
|
+
AttributeName.anName,
|
|
722
|
+
self._dequote_string(guid_token.value),
|
|
723
|
+
)
|
|
724
|
+
type_node.add_child(guid)
|
|
725
|
+
self._accept("]")
|
|
726
|
+
elif self._at("["):
|
|
727
|
+
attributes = self._parse_attributes()
|
|
728
|
+
if self._at("property"):
|
|
729
|
+
prop = self._parse_property()
|
|
730
|
+
prop.add_child(attributes)
|
|
731
|
+
active_parent.add_child(prop)
|
|
732
|
+
elif self._normalized() in _ROUTINES or (
|
|
733
|
+
self._normalized() == "class"
|
|
734
|
+
and self._normalized(1) in _ROUTINES
|
|
735
|
+
):
|
|
736
|
+
method = self._parse_routine(parse_body=False)
|
|
737
|
+
method.add_child(attributes)
|
|
738
|
+
active_parent.add_child(method)
|
|
739
|
+
elif self._is_name(allow_keywords=True):
|
|
740
|
+
fields = self._parse_named_declaration(
|
|
741
|
+
SyntaxNodeType.ntField,
|
|
742
|
+
attributes=attributes,
|
|
743
|
+
)
|
|
744
|
+
for field in fields:
|
|
745
|
+
if class_fields:
|
|
746
|
+
field.set_attribute(AttributeName.anClass, "true")
|
|
747
|
+
active_parent.add_child(field)
|
|
748
|
+
elif self._is_name(allow_keywords=True):
|
|
749
|
+
fields = self._parse_named_declaration(SyntaxNodeType.ntField)
|
|
750
|
+
for field in fields:
|
|
751
|
+
if class_fields:
|
|
752
|
+
field.set_attribute(AttributeName.anClass, "true")
|
|
753
|
+
active_parent.add_child(field)
|
|
754
|
+
else:
|
|
755
|
+
if self._peek().kind is TokenKind.UNKNOWN:
|
|
756
|
+
self._problem(f"Unsupported type member token {self._peek().value!r}")
|
|
757
|
+
self._advance()
|
|
758
|
+
self._ensure_progress(before)
|
|
759
|
+
self._accept("end")
|
|
760
|
+
|
|
761
|
+
def _parse_variant_part(
|
|
762
|
+
self,
|
|
763
|
+
type_node: SyntaxNode,
|
|
764
|
+
*,
|
|
765
|
+
stops: frozenset[str] = frozenset({"end"}),
|
|
766
|
+
) -> None:
|
|
767
|
+
self._advance()
|
|
768
|
+
selector_tokens = self._collect_until({"of"})
|
|
769
|
+
self._accept("of")
|
|
770
|
+
colon = self._top_level_token_index(selector_tokens, ":")
|
|
771
|
+
selector_type = selector_tokens if colon is None else selector_tokens[colon + 1 :]
|
|
772
|
+
if selector_type:
|
|
773
|
+
selector = self._node(SyntaxNodeType.ntType, selector_type[0])
|
|
774
|
+
selector.set_attribute(
|
|
775
|
+
AttributeName.anName,
|
|
776
|
+
self._tokens_text(selector_type).strip(),
|
|
777
|
+
)
|
|
778
|
+
type_node.add_child(selector)
|
|
779
|
+
|
|
780
|
+
while not self._eof() and self._normalized() not in stops:
|
|
781
|
+
before = self.index
|
|
782
|
+
label_tokens = self._collect_until({":", ";", "end"})
|
|
783
|
+
if not label_tokens:
|
|
784
|
+
self._accept(";")
|
|
785
|
+
self._ensure_progress(before)
|
|
786
|
+
continue
|
|
787
|
+
labels = self._node(SyntaxNodeType.ntCaseLabels, label_tokens[0])
|
|
788
|
+
for part in self._split_top_level_tokens(label_tokens, {","}):
|
|
789
|
+
expression = self._expression_direct(part)
|
|
790
|
+
if expression is None:
|
|
791
|
+
continue
|
|
792
|
+
label = self._node(SyntaxNodeType.ntCaseLabel, part[0])
|
|
793
|
+
label.add_child(expression)
|
|
794
|
+
labels.add_child(label)
|
|
795
|
+
type_node.add_child(labels)
|
|
796
|
+
self._accept(":")
|
|
797
|
+
if self._accept("("):
|
|
798
|
+
self._parse_variant_fields(type_node)
|
|
799
|
+
self._accept(")")
|
|
800
|
+
self._accept(";")
|
|
801
|
+
self._ensure_progress(before)
|
|
802
|
+
|
|
803
|
+
def _parse_variant_fields(self, type_node: SyntaxNode) -> None:
|
|
804
|
+
while not self._eof() and not self._at(")"):
|
|
805
|
+
before = self.index
|
|
806
|
+
attributes = self._parse_attributes() if self._at("[") else None
|
|
807
|
+
if self._at("case"):
|
|
808
|
+
self._parse_variant_part(type_node, stops=frozenset({")"}))
|
|
809
|
+
elif self._is_name(allow_keywords=True):
|
|
810
|
+
fields = self._parse_named_declaration(
|
|
811
|
+
SyntaxNodeType.ntField,
|
|
812
|
+
attributes=attributes,
|
|
813
|
+
)
|
|
814
|
+
for field in fields:
|
|
815
|
+
type_node.add_child(field)
|
|
816
|
+
else:
|
|
817
|
+
self._advance()
|
|
818
|
+
self._accept(";")
|
|
819
|
+
self._ensure_progress(before)
|
|
820
|
+
|
|
821
|
+
def _parse_attributes(self) -> SyntaxNode:
|
|
822
|
+
attributes = self._node(SyntaxNodeType.ntAttributes, self._peek())
|
|
823
|
+
while self._accept("["):
|
|
824
|
+
while not self._eof() and not self._at("]"):
|
|
825
|
+
before = self.index
|
|
826
|
+
token = self._peek()
|
|
827
|
+
name = self._read_qualified_name(allow_keywords=True)
|
|
828
|
+
if name:
|
|
829
|
+
attribute = self._node(SyntaxNodeType.ntAttribute, token)
|
|
830
|
+
attribute.set_attribute(AttributeName.anName, name)
|
|
831
|
+
if self._accept("("):
|
|
832
|
+
argument_tokens = self._collect_until({")"})
|
|
833
|
+
arguments = self._node(SyntaxNodeType.ntArguments, self._peek(-1))
|
|
834
|
+
for part in self._split_top_level_tokens(argument_tokens, {","}):
|
|
835
|
+
expression = self._expression_direct(part)
|
|
836
|
+
if expression is not None:
|
|
837
|
+
argument = self._node(
|
|
838
|
+
SyntaxNodeType.ntPositionalArgument,
|
|
839
|
+
part[0],
|
|
840
|
+
)
|
|
841
|
+
argument.add_child(expression)
|
|
842
|
+
arguments.add_child(argument)
|
|
843
|
+
self._accept(")")
|
|
844
|
+
attribute.add_child(arguments)
|
|
845
|
+
attributes.add_child(attribute)
|
|
846
|
+
else:
|
|
847
|
+
self._advance()
|
|
848
|
+
self._accept(",")
|
|
849
|
+
self._ensure_progress(before)
|
|
850
|
+
self._accept("]")
|
|
851
|
+
return attributes
|
|
852
|
+
|
|
853
|
+
def _parse_enum(self) -> SyntaxNode:
|
|
854
|
+
token = self._advance()
|
|
855
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
856
|
+
type_node.set_attribute(AttributeName.anName, "enum")
|
|
857
|
+
while not self._eof() and not self._at(")"):
|
|
858
|
+
before = self.index
|
|
859
|
+
item_token = self._peek()
|
|
860
|
+
name = self._read_name()
|
|
861
|
+
if name:
|
|
862
|
+
item = self._node(SyntaxNodeType.ntEnum, item_token)
|
|
863
|
+
item.add_child(self._valued(SyntaxNodeType.ntName, item_token, name))
|
|
864
|
+
if self._accept("=") or self._accept(":="):
|
|
865
|
+
value_tokens = self._collect_until({",", ")"})
|
|
866
|
+
if value_tokens:
|
|
867
|
+
value = self._node(SyntaxNodeType.ntValue, value_tokens[0])
|
|
868
|
+
expression = self._expression_direct(value_tokens)
|
|
869
|
+
if expression is not None:
|
|
870
|
+
value.add_child(expression)
|
|
871
|
+
item.add_child(value)
|
|
872
|
+
else:
|
|
873
|
+
self._problem(
|
|
874
|
+
"Expected enum value after '='",
|
|
875
|
+
item_token,
|
|
876
|
+
)
|
|
877
|
+
type_node.add_child(item)
|
|
878
|
+
else:
|
|
879
|
+
self._advance()
|
|
880
|
+
self._accept(",")
|
|
881
|
+
self._ensure_progress(before)
|
|
882
|
+
self._accept(")")
|
|
883
|
+
return type_node
|
|
884
|
+
|
|
885
|
+
def _parse_array_type(self, stops: set[str] | None = None) -> SyntaxNode:
|
|
886
|
+
token = self._advance()
|
|
887
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
888
|
+
type_node.set_attribute(AttributeName.anType, "array")
|
|
889
|
+
if self._accept("["):
|
|
890
|
+
bounds = self._node(SyntaxNodeType.ntBounds, self._peek(-1))
|
|
891
|
+
while not self._eof() and not self._at("]"):
|
|
892
|
+
bound_token = self._peek()
|
|
893
|
+
values = self._collect_until({",", "]"})
|
|
894
|
+
bound = self._valued(
|
|
895
|
+
SyntaxNodeType.ntDimension,
|
|
896
|
+
bound_token,
|
|
897
|
+
'',
|
|
898
|
+
)
|
|
899
|
+
bound.value, bound.source_positions = (
|
|
900
|
+
self._tokens_text_with_positions(values)
|
|
901
|
+
)
|
|
902
|
+
if values:
|
|
903
|
+
bound.token_end_line, bound.token_end_col = (
|
|
904
|
+
self._token_end_position(values[-1])
|
|
905
|
+
)
|
|
906
|
+
bounds.add_child(bound)
|
|
907
|
+
self._accept(",")
|
|
908
|
+
self._accept("]")
|
|
909
|
+
type_node.add_child(bounds)
|
|
910
|
+
self._accept("of")
|
|
911
|
+
element_stops = {";", ")", ","}
|
|
912
|
+
if stops:
|
|
913
|
+
element_stops.update(stops)
|
|
914
|
+
type_node.add_child(self._parse_type_annotation(element_stops))
|
|
915
|
+
return type_node
|
|
916
|
+
|
|
917
|
+
def _parse_procedural_type(self) -> SyntaxNode:
|
|
918
|
+
token = self._advance()
|
|
919
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
920
|
+
type_node.set_attribute(AttributeName.anType, token.normalized)
|
|
921
|
+
if self._at("("):
|
|
922
|
+
type_node.add_child(self._parse_parameters())
|
|
923
|
+
if token.normalized == "function" and self._accept(":"):
|
|
924
|
+
result = self._node(SyntaxNodeType.ntReturnType, self._peek())
|
|
925
|
+
result_type = self._parse_type_reference({";"})
|
|
926
|
+
result_name = result_type.get_attribute(AttributeName.anName)
|
|
927
|
+
for suffix, kind in ((" of object", "object"), (" is nested", "nested")):
|
|
928
|
+
if result_name.casefold().endswith(suffix):
|
|
929
|
+
result_type.set_attribute(
|
|
930
|
+
AttributeName.anName,
|
|
931
|
+
result_name[:-len(suffix)].rstrip(),
|
|
932
|
+
)
|
|
933
|
+
type_node.set_attribute(AttributeName.anKind, kind)
|
|
934
|
+
break
|
|
935
|
+
result.add_child(result_type)
|
|
936
|
+
type_node.add_child(result)
|
|
937
|
+
if self._accept("of"):
|
|
938
|
+
if self._accept("object"):
|
|
939
|
+
type_node.set_attribute(AttributeName.anKind, "object")
|
|
940
|
+
elif self._accept("is"):
|
|
941
|
+
if self._accept("nested"):
|
|
942
|
+
type_node.set_attribute(AttributeName.anKind, "nested")
|
|
943
|
+
self._accept(";")
|
|
944
|
+
while self._normalized() in _CALLING_CONVENTIONS | {
|
|
945
|
+
"cblock",
|
|
946
|
+
"extdecl",
|
|
947
|
+
"varargs",
|
|
948
|
+
}:
|
|
949
|
+
directive = self._advance().normalized
|
|
950
|
+
if directive == "varargs":
|
|
951
|
+
type_node.set_attribute(AttributeName.anVarArgs, "true")
|
|
952
|
+
else:
|
|
953
|
+
type_node.set_attribute(AttributeName.anCallingConvention, directive)
|
|
954
|
+
self._accept(";")
|
|
955
|
+
return type_node
|
|
956
|
+
|
|
957
|
+
def _parse_type_reference(self, stops: set[str]) -> SyntaxNode:
|
|
958
|
+
token = self._peek()
|
|
959
|
+
values = self._collect_until(stops)
|
|
960
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
961
|
+
name, positions = self._tokens_text_with_positions(values)
|
|
962
|
+
name = name.strip()
|
|
963
|
+
if name:
|
|
964
|
+
type_node.set_attribute(AttributeName.anName, name)
|
|
965
|
+
type_node.source_positions = positions
|
|
966
|
+
if values:
|
|
967
|
+
type_node.token_end_line, type_node.token_end_col = (
|
|
968
|
+
self._token_end_position(values[-1])
|
|
969
|
+
)
|
|
970
|
+
return type_node
|
|
971
|
+
|
|
972
|
+
def _parse_constants(self, kind: str, *, nested: bool = False) -> SyntaxNode:
|
|
973
|
+
section = self._node(SyntaxNodeType.ntConstants, self._advance())
|
|
974
|
+
section.set_attribute(AttributeName.anKind, kind)
|
|
975
|
+
while not self._eof():
|
|
976
|
+
before = self.index
|
|
977
|
+
if self._peek().kind is TokenKind.KEYWORD and (
|
|
978
|
+
self._normalized() in _SECTIONS
|
|
979
|
+
or self._normalized() in _ROUTINES
|
|
980
|
+
):
|
|
981
|
+
break
|
|
982
|
+
if nested and (
|
|
983
|
+
self._normalized() in _VISIBILITIES
|
|
984
|
+
or self._normalized() in {"strict", "property"}
|
|
985
|
+
or (
|
|
986
|
+
self._normalized() == "class"
|
|
987
|
+
and self._normalized(1)
|
|
988
|
+
in _ROUTINES | {"const", "property", "var"}
|
|
989
|
+
)
|
|
990
|
+
):
|
|
991
|
+
break
|
|
992
|
+
token = self._peek()
|
|
993
|
+
name = self._read_name()
|
|
994
|
+
if not name:
|
|
995
|
+
self._advance()
|
|
996
|
+
self._ensure_progress(before)
|
|
997
|
+
continue
|
|
998
|
+
constant = self._node(SyntaxNodeType.ntConstant, token)
|
|
999
|
+
constant.add_child(self._valued(SyntaxNodeType.ntName, token, name))
|
|
1000
|
+
if self._accept(":"):
|
|
1001
|
+
constant.add_child(self._parse_type_annotation({"=", ";"}))
|
|
1002
|
+
if self._accept("="):
|
|
1003
|
+
value_token = self._peek()
|
|
1004
|
+
value = self._collect_until({";"}, max_retained=256)
|
|
1005
|
+
value_node = self._node(SyntaxNodeType.ntValue, token)
|
|
1006
|
+
expression = self._expression_direct(value)
|
|
1007
|
+
if expression is not None:
|
|
1008
|
+
value_node.add_child(expression)
|
|
1009
|
+
elif value:
|
|
1010
|
+
value_node.add_child(
|
|
1011
|
+
self._valued(
|
|
1012
|
+
SyntaxNodeType.ntLiteral,
|
|
1013
|
+
value_token,
|
|
1014
|
+
self._tokens_text(value),
|
|
1015
|
+
)
|
|
1016
|
+
)
|
|
1017
|
+
constant.add_child(value_node)
|
|
1018
|
+
section.add_child(constant)
|
|
1019
|
+
self._consume_through(";")
|
|
1020
|
+
self._ensure_progress(before)
|
|
1021
|
+
return section
|
|
1022
|
+
|
|
1023
|
+
def _parse_type_annotation(self, stops: set[str]) -> SyntaxNode:
|
|
1024
|
+
token = self._peek()
|
|
1025
|
+
keyword = not self._source_token_value(token).startswith('&')
|
|
1026
|
+
if keyword and self._normalized() in {
|
|
1027
|
+
"packed",
|
|
1028
|
+
"bitpacked",
|
|
1029
|
+
"class",
|
|
1030
|
+
"record",
|
|
1031
|
+
"object",
|
|
1032
|
+
"interface",
|
|
1033
|
+
"dispinterface",
|
|
1034
|
+
"objcclass",
|
|
1035
|
+
"objcprotocol",
|
|
1036
|
+
"objccategory",
|
|
1037
|
+
}:
|
|
1038
|
+
return self._parse_type()
|
|
1039
|
+
if keyword and self._accept("set"):
|
|
1040
|
+
type_node = self._node(SyntaxNodeType.ntType, token)
|
|
1041
|
+
type_node.set_attribute(AttributeName.anType, "set")
|
|
1042
|
+
self._accept("of")
|
|
1043
|
+
type_node.add_child(self._parse_type_reference(stops))
|
|
1044
|
+
return type_node
|
|
1045
|
+
if keyword and self._at("array"):
|
|
1046
|
+
return self._parse_array_type(stops)
|
|
1047
|
+
if (
|
|
1048
|
+
(keyword and self._normalized() in _ROUTINES | {"file"})
|
|
1049
|
+
or self._at("^")
|
|
1050
|
+
or (
|
|
1051
|
+
keyword
|
|
1052
|
+
and self._at("reference")
|
|
1053
|
+
and self._normalized(1) == "to"
|
|
1054
|
+
and self._normalized(2) in _ROUTINES
|
|
1055
|
+
)
|
|
1056
|
+
):
|
|
1057
|
+
return self._parse_type()
|
|
1058
|
+
return self._parse_type_reference(stops)
|
|
1059
|
+
|
|
1060
|
+
def _parse_variables(self, kind: str) -> SyntaxNode:
|
|
1061
|
+
section = self._node(SyntaxNodeType.ntVariables, self._advance())
|
|
1062
|
+
section.set_attribute(AttributeName.anKind, kind)
|
|
1063
|
+
while not self._eof():
|
|
1064
|
+
before = self.index
|
|
1065
|
+
word = self._normalized()
|
|
1066
|
+
if self._peek().kind is TokenKind.KEYWORD and (
|
|
1067
|
+
word in _SECTIONS
|
|
1068
|
+
or word in _ROUTINES
|
|
1069
|
+
or (word == "class" and self._normalized(1) in _ROUTINES)
|
|
1070
|
+
):
|
|
1071
|
+
break
|
|
1072
|
+
if not self._is_name(allow_keywords=True):
|
|
1073
|
+
self._advance()
|
|
1074
|
+
self._ensure_progress(before)
|
|
1075
|
+
continue
|
|
1076
|
+
for variable in self._parse_named_declaration(SyntaxNodeType.ntVariable):
|
|
1077
|
+
section.add_child(variable)
|
|
1078
|
+
self._ensure_progress(before)
|
|
1079
|
+
return section
|
|
1080
|
+
|
|
1081
|
+
def _parse_named_declaration(
|
|
1082
|
+
self,
|
|
1083
|
+
typ: SyntaxNodeType,
|
|
1084
|
+
*,
|
|
1085
|
+
attributes: SyntaxNode | None = None,
|
|
1086
|
+
) -> list[SyntaxNode]:
|
|
1087
|
+
names: list[tuple[Token, str]] = []
|
|
1088
|
+
while not self._eof():
|
|
1089
|
+
token = self._peek()
|
|
1090
|
+
name = self._read_name()
|
|
1091
|
+
if not name:
|
|
1092
|
+
break
|
|
1093
|
+
names.append((token, name))
|
|
1094
|
+
if not self._accept(","):
|
|
1095
|
+
break
|
|
1096
|
+
if not self._accept(":"):
|
|
1097
|
+
self._problem("Expected ':' in declaration")
|
|
1098
|
+
self._consume_through(";")
|
|
1099
|
+
result: list[SyntaxNode] = []
|
|
1100
|
+
for token, name in names:
|
|
1101
|
+
node = self._node(typ, token)
|
|
1102
|
+
node.add_child(self._valued(SyntaxNodeType.ntName, token, name))
|
|
1103
|
+
if attributes is not None:
|
|
1104
|
+
node.add_child(attributes.clone())
|
|
1105
|
+
result.append(node)
|
|
1106
|
+
return result
|
|
1107
|
+
type_stops = {";", ")", "=", "absolute"}
|
|
1108
|
+
if typ is SyntaxNodeType.ntField:
|
|
1109
|
+
type_stops.add("end")
|
|
1110
|
+
type_node = self._parse_type_annotation(type_stops)
|
|
1111
|
+
if self._accept("=") or self._accept("absolute"):
|
|
1112
|
+
self._skip_balanced_until({";"})
|
|
1113
|
+
self._accept(";")
|
|
1114
|
+
declaration_attributes: list[tuple[AttributeName, str]] = []
|
|
1115
|
+
type_name = type_node.get_attribute(AttributeName.anName).lstrip().casefold()
|
|
1116
|
+
procedural_field = typ is SyntaxNodeType.ntField and type_name.startswith(
|
|
1117
|
+
("function", "procedure", "reference to")
|
|
1118
|
+
)
|
|
1119
|
+
while self._normalized() in _DECLARATION_MODIFIERS:
|
|
1120
|
+
if typ is SyntaxNodeType.ntField and not (
|
|
1121
|
+
self._normalized()
|
|
1122
|
+
in {"deprecated", "experimental", "library", "platform"}
|
|
1123
|
+
or (
|
|
1124
|
+
procedural_field
|
|
1125
|
+
and self._normalized() in _CALLING_CONVENTIONS
|
|
1126
|
+
| {"extdecl", "far", "near", "varargs"}
|
|
1127
|
+
)
|
|
1128
|
+
):
|
|
1129
|
+
break
|
|
1130
|
+
modifier = self._advance().normalized
|
|
1131
|
+
if modifier == "cvar":
|
|
1132
|
+
declaration_attributes.append((AttributeName.anKind, modifier))
|
|
1133
|
+
elif modifier in _CALLING_CONVENTIONS or modifier == "extdecl":
|
|
1134
|
+
declaration_attributes.append(
|
|
1135
|
+
(AttributeName.anCallingConvention, modifier)
|
|
1136
|
+
)
|
|
1137
|
+
elif modifier == "varargs":
|
|
1138
|
+
declaration_attributes.append((AttributeName.anVarArgs, "true"))
|
|
1139
|
+
elif modifier == "far":
|
|
1140
|
+
declaration_attributes.append((AttributeName.anFar, "true"))
|
|
1141
|
+
elif modifier == "near":
|
|
1142
|
+
declaration_attributes.append((AttributeName.anNear, "true"))
|
|
1143
|
+
elif modifier in {"external", "weakexternal"}:
|
|
1144
|
+
declaration_attributes.append((AttributeName.anExternal, "true"))
|
|
1145
|
+
elif modifier in {"export", "public"}:
|
|
1146
|
+
declaration_attributes.append((AttributeName.anExport, "true"))
|
|
1147
|
+
elif modifier == "deprecated":
|
|
1148
|
+
declaration_attributes.append((AttributeName.anDeprecated, "true"))
|
|
1149
|
+
else:
|
|
1150
|
+
declaration_attributes.append((AttributeName.anKind, modifier))
|
|
1151
|
+
if not self._at(";"):
|
|
1152
|
+
self._skip_balanced_until({";"})
|
|
1153
|
+
self._accept(";")
|
|
1154
|
+
result: list[SyntaxNode] = []
|
|
1155
|
+
for token, name in names:
|
|
1156
|
+
node = self._node(typ, token)
|
|
1157
|
+
node.add_child(self._valued(SyntaxNodeType.ntName, token, name))
|
|
1158
|
+
node.add_child(type_node.clone())
|
|
1159
|
+
if attributes is not None:
|
|
1160
|
+
node.add_child(attributes.clone())
|
|
1161
|
+
for attribute, value in declaration_attributes:
|
|
1162
|
+
node.set_attribute(attribute, value)
|
|
1163
|
+
for attribute in (
|
|
1164
|
+
AttributeName.anCallingConvention,
|
|
1165
|
+
AttributeName.anVarArgs,
|
|
1166
|
+
):
|
|
1167
|
+
value = type_node.get_attribute(attribute)
|
|
1168
|
+
if value:
|
|
1169
|
+
node.set_attribute(attribute, value)
|
|
1170
|
+
result.append(node)
|
|
1171
|
+
return result
|
|
1172
|
+
|
|
1173
|
+
def _parse_property(self) -> SyntaxNode:
|
|
1174
|
+
token = self._advance()
|
|
1175
|
+
node = self._node(SyntaxNodeType.ntProperty, token)
|
|
1176
|
+
name = self._read_name()
|
|
1177
|
+
node.set_attribute(AttributeName.anName, name)
|
|
1178
|
+
if self._at("["):
|
|
1179
|
+
for parameter in self._parse_property_parameters():
|
|
1180
|
+
node.add_child(parameter)
|
|
1181
|
+
if self._accept(":"):
|
|
1182
|
+
node.add_child(self._parse_type_reference({";", *_PROPERTY_SPECIFIERS}))
|
|
1183
|
+
|
|
1184
|
+
while not self._eof():
|
|
1185
|
+
before = self.index
|
|
1186
|
+
word = self._normalized()
|
|
1187
|
+
if word == ";":
|
|
1188
|
+
self._advance()
|
|
1189
|
+
if self._normalized() not in _PROPERTY_SPECIFIERS | {
|
|
1190
|
+
"deprecated",
|
|
1191
|
+
"experimental",
|
|
1192
|
+
"library",
|
|
1193
|
+
"platform",
|
|
1194
|
+
}:
|
|
1195
|
+
break
|
|
1196
|
+
continue
|
|
1197
|
+
if word == "index":
|
|
1198
|
+
specifier = self._node(SyntaxNodeType.ntIndex, self._advance())
|
|
1199
|
+
expression = self._property_expression()
|
|
1200
|
+
if expression is not None:
|
|
1201
|
+
specifier.add_child(expression)
|
|
1202
|
+
node.add_child(specifier)
|
|
1203
|
+
elif word in _PROPERTY_ACCESSORS:
|
|
1204
|
+
specifier_token = self._advance()
|
|
1205
|
+
if word == "read":
|
|
1206
|
+
typ = SyntaxNodeType.ntRead
|
|
1207
|
+
elif word == "write":
|
|
1208
|
+
typ = SyntaxNodeType.ntWrite
|
|
1209
|
+
else:
|
|
1210
|
+
typ = SyntaxNodeType.ntUnknown
|
|
1211
|
+
specifier = self._node(typ, specifier_token)
|
|
1212
|
+
if word not in {"read", "write"}:
|
|
1213
|
+
specifier.set_attribute(AttributeName.anKind, word)
|
|
1214
|
+
if word not in {"readonly", "writeonly"}:
|
|
1215
|
+
expression = self._property_expression()
|
|
1216
|
+
if expression is not None:
|
|
1217
|
+
specifier.add_child(expression)
|
|
1218
|
+
node.add_child(specifier)
|
|
1219
|
+
elif word in _PROPERTY_STORAGE:
|
|
1220
|
+
specifier_token = self._advance()
|
|
1221
|
+
specifier = self._node(
|
|
1222
|
+
SyntaxNodeType.ntDefault
|
|
1223
|
+
if word in {"default", "nodefault"}
|
|
1224
|
+
else SyntaxNodeType.ntUnknown,
|
|
1225
|
+
specifier_token,
|
|
1226
|
+
)
|
|
1227
|
+
if word == "nodefault" or word == "stored":
|
|
1228
|
+
specifier.set_attribute(AttributeName.anKind, word)
|
|
1229
|
+
if word != "nodefault" and not self._at(";"):
|
|
1230
|
+
expression = self._property_expression()
|
|
1231
|
+
if expression is not None:
|
|
1232
|
+
specifier.add_child(expression)
|
|
1233
|
+
node.add_child(specifier)
|
|
1234
|
+
elif word in {"dispid", "implements"}:
|
|
1235
|
+
specifier_token = self._advance()
|
|
1236
|
+
typ = (
|
|
1237
|
+
SyntaxNodeType.ntImplements
|
|
1238
|
+
if word == "implements"
|
|
1239
|
+
else SyntaxNodeType.ntUnknown
|
|
1240
|
+
)
|
|
1241
|
+
specifier = self._node(typ, specifier_token)
|
|
1242
|
+
if word == "dispid":
|
|
1243
|
+
specifier.set_attribute(AttributeName.anKind, word)
|
|
1244
|
+
expression = self._property_expression()
|
|
1245
|
+
if expression is not None:
|
|
1246
|
+
specifier.add_child(expression)
|
|
1247
|
+
node.add_child(specifier)
|
|
1248
|
+
elif word in {"deprecated", "experimental", "library", "platform"}:
|
|
1249
|
+
self._advance()
|
|
1250
|
+
if word == "deprecated":
|
|
1251
|
+
value = "true"
|
|
1252
|
+
if self._peek().kind is TokenKind.STRING:
|
|
1253
|
+
value = self._dequote_string(self._advance().value)
|
|
1254
|
+
node.set_attribute(AttributeName.anDeprecated, value)
|
|
1255
|
+
else:
|
|
1256
|
+
node.set_attribute(AttributeName.anKind, word)
|
|
1257
|
+
else:
|
|
1258
|
+
self._problem(f"Unsupported property specifier {self._peek().value!r}")
|
|
1259
|
+
self._skip_balanced_until({";"})
|
|
1260
|
+
self._ensure_progress(before)
|
|
1261
|
+
return node
|
|
1262
|
+
|
|
1263
|
+
def _parse_property_parameters(self) -> list[SyntaxNode]:
|
|
1264
|
+
self._advance()
|
|
1265
|
+
parameters: list[SyntaxNode] = []
|
|
1266
|
+
while not self._eof() and not self._at("]"):
|
|
1267
|
+
before = self.index
|
|
1268
|
+
modifier = ""
|
|
1269
|
+
if self._normalized() in {"const", "var", "out", "constref"}:
|
|
1270
|
+
modifier = self._advance().normalized
|
|
1271
|
+
attributes = self._parse_attributes() if self._at("[") else None
|
|
1272
|
+
names: list[tuple[Token, str]] = []
|
|
1273
|
+
while self._is_name(allow_keywords=True):
|
|
1274
|
+
name_token = self._peek()
|
|
1275
|
+
names.append((name_token, self._read_name()))
|
|
1276
|
+
if not self._accept(","):
|
|
1277
|
+
break
|
|
1278
|
+
type_node = None
|
|
1279
|
+
if self._accept(":"):
|
|
1280
|
+
type_node = self._parse_type_reference({";", "]", "="})
|
|
1281
|
+
default_tokens: list[Token] = []
|
|
1282
|
+
if self._accept("="):
|
|
1283
|
+
default_tokens = self._collect_until({";", "]"})
|
|
1284
|
+
for name_token, name in names:
|
|
1285
|
+
parameter = self._node(SyntaxNodeType.ntParameter, name_token)
|
|
1286
|
+
if modifier:
|
|
1287
|
+
parameter.set_attribute(AttributeName.anKind, modifier)
|
|
1288
|
+
parameter.add_child(
|
|
1289
|
+
self._valued(SyntaxNodeType.ntName, name_token, name)
|
|
1290
|
+
)
|
|
1291
|
+
if type_node is not None:
|
|
1292
|
+
parameter.add_child(type_node.clone())
|
|
1293
|
+
if attributes is not None:
|
|
1294
|
+
parameter.add_child(attributes.clone())
|
|
1295
|
+
if default_tokens:
|
|
1296
|
+
value = self._node(SyntaxNodeType.ntValue, default_tokens[0])
|
|
1297
|
+
expression = self._expression_direct(default_tokens)
|
|
1298
|
+
if expression is not None:
|
|
1299
|
+
value.add_child(expression)
|
|
1300
|
+
parameter.add_child(value)
|
|
1301
|
+
parameters.append(parameter)
|
|
1302
|
+
self._accept(";")
|
|
1303
|
+
self._ensure_progress(before)
|
|
1304
|
+
self._accept("]")
|
|
1305
|
+
return parameters
|
|
1306
|
+
|
|
1307
|
+
def _property_expression(self) -> SyntaxNode | None:
|
|
1308
|
+
values = self._collect_until({";", *_PROPERTY_SPECIFIERS})
|
|
1309
|
+
return self._expression_direct(values)
|
|
1310
|
+
|
|
1311
|
+
def _expression_direct(self, values: list[Token]) -> SyntaxNode | None:
|
|
1312
|
+
expression = self._expression_node(values)
|
|
1313
|
+
if expression is not None and len(expression.child_nodes) == 1:
|
|
1314
|
+
child = expression.child_nodes[0]
|
|
1315
|
+
expression.extract_child(child)
|
|
1316
|
+
return child
|
|
1317
|
+
return expression
|
|
1318
|
+
|
|
1319
|
+
def _parse_routine(self, *, parse_body: bool) -> SyntaxNode:
|
|
1320
|
+
class_token: Token | None = None
|
|
1321
|
+
if self._at("class"):
|
|
1322
|
+
class_token = self._advance()
|
|
1323
|
+
kind_token = self._advance()
|
|
1324
|
+
node = self._node(SyntaxNodeType.ntMethod, class_token or kind_token, compound=True)
|
|
1325
|
+
node.set_attribute(AttributeName.anKind, kind_token.normalized)
|
|
1326
|
+
if class_token is not None:
|
|
1327
|
+
node.set_attribute(AttributeName.anClass, "true")
|
|
1328
|
+
if (
|
|
1329
|
+
kind_token.normalized == "operator"
|
|
1330
|
+
and self._peek().kind is TokenKind.SYMBOL
|
|
1331
|
+
):
|
|
1332
|
+
name = self._advance().value
|
|
1333
|
+
else:
|
|
1334
|
+
name = self._read_routine_name()
|
|
1335
|
+
node.set_attribute(AttributeName.anName, name)
|
|
1336
|
+
self._parse_type_parameters(node)
|
|
1337
|
+
if self._at("("):
|
|
1338
|
+
node.add_child(self._parse_parameters())
|
|
1339
|
+
if kind_token.normalized in {"function", "operator"} and self._accept(":"):
|
|
1340
|
+
result = self._node(SyntaxNodeType.ntReturnType, self._peek())
|
|
1341
|
+
result.add_child(self._parse_type_reference({";"}))
|
|
1342
|
+
node.add_child(result)
|
|
1343
|
+
if self._accept("="):
|
|
1344
|
+
target_token = self._peek()
|
|
1345
|
+
target = self._read_routine_name()
|
|
1346
|
+
resolution = self._node(
|
|
1347
|
+
SyntaxNodeType.ntResolutionClause,
|
|
1348
|
+
target_token,
|
|
1349
|
+
)
|
|
1350
|
+
resolution.set_attribute(AttributeName.anName, target)
|
|
1351
|
+
node.add_child(resolution)
|
|
1352
|
+
self._consume_through(";")
|
|
1353
|
+
return node
|
|
1354
|
+
self._accept(";")
|
|
1355
|
+
self._parse_method_directives(node)
|
|
1356
|
+
if not parse_body:
|
|
1357
|
+
if self._at("asm"):
|
|
1358
|
+
asm = self._node(
|
|
1359
|
+
SyntaxNodeType.ntStatements,
|
|
1360
|
+
self._advance(),
|
|
1361
|
+
compound=True,
|
|
1362
|
+
)
|
|
1363
|
+
asm.set_attribute(AttributeName.anKind, "asm")
|
|
1364
|
+
self._synchronize({"end"})
|
|
1365
|
+
self._accept("end")
|
|
1366
|
+
self._accept(";")
|
|
1367
|
+
node.add_child(asm)
|
|
1368
|
+
return node
|
|
1369
|
+
if any(
|
|
1370
|
+
node.has_attribute(attribute)
|
|
1371
|
+
for attribute in (
|
|
1372
|
+
AttributeName.anAbstract,
|
|
1373
|
+
AttributeName.anExternal,
|
|
1374
|
+
AttributeName.anForwarded,
|
|
1375
|
+
)
|
|
1376
|
+
):
|
|
1377
|
+
return node
|
|
1378
|
+
|
|
1379
|
+
while (
|
|
1380
|
+
self._normalized() in {"label", "const", "type", "var", "threadvar"}
|
|
1381
|
+
or self._normalized() in _ROUTINES
|
|
1382
|
+
or (
|
|
1383
|
+
self._normalized() == "class"
|
|
1384
|
+
and self._normalized(1) in _ROUTINES
|
|
1385
|
+
)
|
|
1386
|
+
):
|
|
1387
|
+
word = self._normalized()
|
|
1388
|
+
if word in {"var", "threadvar"}:
|
|
1389
|
+
node.add_child(self._parse_variables(word))
|
|
1390
|
+
elif word in {"const"}:
|
|
1391
|
+
node.add_child(self._parse_constants(word))
|
|
1392
|
+
elif word == "type":
|
|
1393
|
+
node.add_child(self._parse_type_section())
|
|
1394
|
+
elif word in _ROUTINES or (
|
|
1395
|
+
word == "class"
|
|
1396
|
+
and self._normalized(1) in _ROUTINES
|
|
1397
|
+
):
|
|
1398
|
+
node.add_child(self._parse_routine(parse_body=True))
|
|
1399
|
+
else:
|
|
1400
|
+
self._consume_through(";")
|
|
1401
|
+
if self._at("begin"):
|
|
1402
|
+
node.add_child(self._parse_begin_block())
|
|
1403
|
+
self._accept(";")
|
|
1404
|
+
elif self._at("asm"):
|
|
1405
|
+
asm = self._node(SyntaxNodeType.ntStatements, self._advance(), compound=True)
|
|
1406
|
+
asm.set_attribute(AttributeName.anKind, "asm")
|
|
1407
|
+
self._synchronize({"end"})
|
|
1408
|
+
self._accept("end")
|
|
1409
|
+
self._accept(";")
|
|
1410
|
+
node.add_child(asm)
|
|
1411
|
+
return node
|
|
1412
|
+
|
|
1413
|
+
def _parse_parameters(self) -> SyntaxNode:
|
|
1414
|
+
start = self._advance()
|
|
1415
|
+
params = self._node(SyntaxNodeType.ntParameters, start)
|
|
1416
|
+
while not self._eof() and not self._at(")"):
|
|
1417
|
+
before = self.index
|
|
1418
|
+
if self._at("..") and self._at(".", 1):
|
|
1419
|
+
self._advance()
|
|
1420
|
+
self._advance()
|
|
1421
|
+
params.set_attribute(AttributeName.anVarArgs, "true")
|
|
1422
|
+
self._accept(";")
|
|
1423
|
+
continue
|
|
1424
|
+
modifier = ""
|
|
1425
|
+
if self._normalized() in {"const", "var", "out", "constref"}:
|
|
1426
|
+
modifier = self._advance().normalized
|
|
1427
|
+
attributes = self._parse_attributes() if self._at("[") else None
|
|
1428
|
+
names: list[tuple[Token, str]] = []
|
|
1429
|
+
while self._is_name(allow_keywords=True):
|
|
1430
|
+
token = self._peek()
|
|
1431
|
+
name = self._read_name()
|
|
1432
|
+
names.append((token, name))
|
|
1433
|
+
if not self._accept(","):
|
|
1434
|
+
break
|
|
1435
|
+
type_node = None
|
|
1436
|
+
if self._accept(":"):
|
|
1437
|
+
type_node = self._parse_type_annotation({";", ")", "="})
|
|
1438
|
+
if self._accept("="):
|
|
1439
|
+
self._skip_balanced_until({";", ")"})
|
|
1440
|
+
for token, name in names:
|
|
1441
|
+
param = self._node(SyntaxNodeType.ntParameter, token)
|
|
1442
|
+
if modifier:
|
|
1443
|
+
param.set_attribute(AttributeName.anKind, modifier)
|
|
1444
|
+
param.add_child(
|
|
1445
|
+
self._valued(SyntaxNodeType.ntName, token, name)
|
|
1446
|
+
)
|
|
1447
|
+
if type_node is not None:
|
|
1448
|
+
param.add_child(type_node.clone())
|
|
1449
|
+
if attributes is not None:
|
|
1450
|
+
param.add_child(attributes.clone())
|
|
1451
|
+
params.add_child(param)
|
|
1452
|
+
self._accept(";")
|
|
1453
|
+
self._ensure_progress(before)
|
|
1454
|
+
self._accept(")")
|
|
1455
|
+
return params
|
|
1456
|
+
|
|
1457
|
+
def _parse_method_directives(self, node: SyntaxNode) -> None:
|
|
1458
|
+
while not self._eof():
|
|
1459
|
+
word = self._normalized()
|
|
1460
|
+
if word in _METHOD_DIRECTIVES:
|
|
1461
|
+
self._advance()
|
|
1462
|
+
attr = _METHOD_DIRECTIVES[word]
|
|
1463
|
+
value = "true"
|
|
1464
|
+
if word == "deprecated" and self._peek().kind is TokenKind.STRING:
|
|
1465
|
+
value = self._dequote_string(self._advance().value)
|
|
1466
|
+
node.set_attribute(attr, value)
|
|
1467
|
+
if word == "external":
|
|
1468
|
+
self._skip_balanced_until({";"})
|
|
1469
|
+
self._accept(";")
|
|
1470
|
+
elif word in {"virtual", "dynamic", "override"}:
|
|
1471
|
+
node.set_attribute(AttributeName.anMethodBinding, self._advance().normalized)
|
|
1472
|
+
self._accept(";")
|
|
1473
|
+
elif word in _CALLING_CONVENTIONS:
|
|
1474
|
+
node.set_attribute(AttributeName.anCallingConvention, self._advance().normalized)
|
|
1475
|
+
self._accept(";")
|
|
1476
|
+
elif word == "nostackframe":
|
|
1477
|
+
self._advance()
|
|
1478
|
+
self._accept(";")
|
|
1479
|
+
elif word == "varargs":
|
|
1480
|
+
node.set_attribute(AttributeName.anVarArgs, "true")
|
|
1481
|
+
self._advance()
|
|
1482
|
+
self._accept(";")
|
|
1483
|
+
elif word in {"message", "dispid"}:
|
|
1484
|
+
token = self._advance()
|
|
1485
|
+
message = self._node(SyntaxNodeType.ntMessage, token)
|
|
1486
|
+
expression = self._expression_direct(self._collect_until({";"}))
|
|
1487
|
+
if expression is not None:
|
|
1488
|
+
message.add_child(expression)
|
|
1489
|
+
node.add_child(message)
|
|
1490
|
+
self._accept(";")
|
|
1491
|
+
elif word in {"experimental", "platform", "library"}:
|
|
1492
|
+
self._advance()
|
|
1493
|
+
self._accept(";")
|
|
1494
|
+
else:
|
|
1495
|
+
break
|
|
1496
|
+
|
|
1497
|
+
def _parse_begin_block(self) -> SyntaxNode:
|
|
1498
|
+
token = self._advance()
|
|
1499
|
+
statements = self._node(SyntaxNodeType.ntStatements, token, compound=True)
|
|
1500
|
+
deferred_end = self._large_flat_block_end(self.index)
|
|
1501
|
+
if deferred_end is None:
|
|
1502
|
+
self._parse_statements(statements, {"end"})
|
|
1503
|
+
else:
|
|
1504
|
+
start = self.index
|
|
1505
|
+
statements._set_child_loader(
|
|
1506
|
+
lambda: self._materialize_deferred_statements(start, deferred_end)
|
|
1507
|
+
)
|
|
1508
|
+
self.index = deferred_end
|
|
1509
|
+
end = self._peek()
|
|
1510
|
+
self._accept("end")
|
|
1511
|
+
if isinstance(statements, CompoundSyntaxNode):
|
|
1512
|
+
statements.end_line = end.line
|
|
1513
|
+
statements.end_col = end.column
|
|
1514
|
+
return statements
|
|
1515
|
+
|
|
1516
|
+
def _large_flat_block_end(self, start: int) -> int | None:
|
|
1517
|
+
index = start
|
|
1518
|
+
while index <= self._last_token_index:
|
|
1519
|
+
token = self.tokens[index]
|
|
1520
|
+
if token.kind is TokenKind.EOF:
|
|
1521
|
+
return None
|
|
1522
|
+
if token.kind is TokenKind.KEYWORD:
|
|
1523
|
+
word = token.normalized
|
|
1524
|
+
if word == "end":
|
|
1525
|
+
return (
|
|
1526
|
+
index
|
|
1527
|
+
if index - start >= _DEFERRED_FLAT_BLOCK_TOKENS
|
|
1528
|
+
else None
|
|
1529
|
+
)
|
|
1530
|
+
if word in _DEFERRED_BLOCK_OPENERS:
|
|
1531
|
+
return None
|
|
1532
|
+
index += 1
|
|
1533
|
+
return None
|
|
1534
|
+
|
|
1535
|
+
def _materialize_deferred_statements(
|
|
1536
|
+
self,
|
|
1537
|
+
start: int,
|
|
1538
|
+
end: int,
|
|
1539
|
+
) -> list[SyntaxNode]:
|
|
1540
|
+
parser = object.__new__(DelphiAstParser)
|
|
1541
|
+
parser.source = self.source
|
|
1542
|
+
parser.file_name = self.file_name
|
|
1543
|
+
parser.build_expressions = self.build_expressions
|
|
1544
|
+
parser.build_node_positions = self.build_node_positions
|
|
1545
|
+
parser.tokens = self.tokens
|
|
1546
|
+
parser._last_token_index = self._last_token_index
|
|
1547
|
+
parser.index = start
|
|
1548
|
+
parser.problems = self.problems
|
|
1549
|
+
parser._node_position_cache = self._node_position_cache
|
|
1550
|
+
parent = SyntaxNode(SyntaxNodeType.ntStatements)
|
|
1551
|
+
parser._parse_statements(parent, {"end"})
|
|
1552
|
+
if parser.index != end:
|
|
1553
|
+
parser._problem("Deferred statement boundary changed")
|
|
1554
|
+
return parent._child_nodes
|
|
1555
|
+
|
|
1556
|
+
def _parse_statements(self, parent: SyntaxNode, stops: set[str]) -> None:
|
|
1557
|
+
while not self._eof() and self._normalized() not in stops:
|
|
1558
|
+
before = self.index
|
|
1559
|
+
statement = self._parse_statement()
|
|
1560
|
+
if statement is not None:
|
|
1561
|
+
parent.add_child(statement)
|
|
1562
|
+
self._accept(";")
|
|
1563
|
+
self._ensure_progress(before)
|
|
1564
|
+
|
|
1565
|
+
def _parse_statement(self) -> SyntaxNode | None:
|
|
1566
|
+
word = self._normalized()
|
|
1567
|
+
if self._at(";"):
|
|
1568
|
+
return self._node(SyntaxNodeType.ntEmptyStatement, self._advance())
|
|
1569
|
+
if self._is_name(allow_keywords=False) and self._at(":", 1):
|
|
1570
|
+
token = self._advance()
|
|
1571
|
+
self._advance()
|
|
1572
|
+
label = self._node(SyntaxNodeType.ntLabel, token)
|
|
1573
|
+
label.set_attribute(AttributeName.anName, token.value)
|
|
1574
|
+
statement = self._parse_statement()
|
|
1575
|
+
if statement is not None:
|
|
1576
|
+
label.add_child(statement)
|
|
1577
|
+
return label
|
|
1578
|
+
if word == "begin":
|
|
1579
|
+
return self._parse_begin_block()
|
|
1580
|
+
control_types = {
|
|
1581
|
+
"if": SyntaxNodeType.ntIf,
|
|
1582
|
+
"for": SyntaxNodeType.ntFor,
|
|
1583
|
+
"while": SyntaxNodeType.ntWhile,
|
|
1584
|
+
"with": SyntaxNodeType.ntWith,
|
|
1585
|
+
}
|
|
1586
|
+
if word in control_types:
|
|
1587
|
+
token = self._advance()
|
|
1588
|
+
node = self._node(control_types[word], token, compound=True)
|
|
1589
|
+
separator = "then" if word == "if" else "do"
|
|
1590
|
+
if word == "for":
|
|
1591
|
+
separator = "do"
|
|
1592
|
+
expression_tokens = self._collect_until(
|
|
1593
|
+
{separator, ";", "end"},
|
|
1594
|
+
track_angles=False,
|
|
1595
|
+
)
|
|
1596
|
+
expression = self._expression_direct(expression_tokens)
|
|
1597
|
+
if expression is not None:
|
|
1598
|
+
node.add_child(expression)
|
|
1599
|
+
self._accept(separator)
|
|
1600
|
+
child = self._parse_statement()
|
|
1601
|
+
if child is not None:
|
|
1602
|
+
node.add_child(child)
|
|
1603
|
+
if word == "if" and self._accept("else"):
|
|
1604
|
+
otherwise = self._node(SyntaxNodeType.ntElse, self._peek(-1))
|
|
1605
|
+
child = self._parse_statement()
|
|
1606
|
+
if child is not None:
|
|
1607
|
+
otherwise.add_child(child)
|
|
1608
|
+
node.add_child(otherwise)
|
|
1609
|
+
return node
|
|
1610
|
+
if word == "repeat":
|
|
1611
|
+
token = self._advance()
|
|
1612
|
+
node = self._node(SyntaxNodeType.ntRepeat, token, compound=True)
|
|
1613
|
+
self._parse_statements(node, {"until"})
|
|
1614
|
+
self._accept("until")
|
|
1615
|
+
expression = self._expression_direct(
|
|
1616
|
+
self._collect_until({";", "end"}, track_angles=False)
|
|
1617
|
+
)
|
|
1618
|
+
if expression is not None:
|
|
1619
|
+
node.add_child(expression)
|
|
1620
|
+
return node
|
|
1621
|
+
if word == "try":
|
|
1622
|
+
return self._parse_try_statement()
|
|
1623
|
+
if word == "case":
|
|
1624
|
+
token = self._advance()
|
|
1625
|
+
node = self._node(SyntaxNodeType.ntCase, token, compound=True)
|
|
1626
|
+
selector = self._node(SyntaxNodeType.ntCaseSelector, token)
|
|
1627
|
+
expression = self._expression_direct(
|
|
1628
|
+
self._collect_until({"of"}, track_angles=False)
|
|
1629
|
+
)
|
|
1630
|
+
if expression is not None:
|
|
1631
|
+
selector.add_child(expression)
|
|
1632
|
+
node.add_child(selector)
|
|
1633
|
+
self._accept("of")
|
|
1634
|
+
while not self._eof() and self._normalized() not in {
|
|
1635
|
+
"else",
|
|
1636
|
+
"otherwise",
|
|
1637
|
+
"end",
|
|
1638
|
+
}:
|
|
1639
|
+
before = self.index
|
|
1640
|
+
label_tokens = self._collect_until(
|
|
1641
|
+
{":", "else", "end"},
|
|
1642
|
+
track_angles=False,
|
|
1643
|
+
)
|
|
1644
|
+
if not self._accept(":"):
|
|
1645
|
+
self._ensure_progress(before)
|
|
1646
|
+
continue
|
|
1647
|
+
labels = self._node(
|
|
1648
|
+
SyntaxNodeType.ntCaseLabels,
|
|
1649
|
+
label_tokens[0] if label_tokens else self._peek(-1),
|
|
1650
|
+
)
|
|
1651
|
+
for part in self._split_top_level_tokens(label_tokens, {","}):
|
|
1652
|
+
if not part:
|
|
1653
|
+
continue
|
|
1654
|
+
label = self._node(SyntaxNodeType.ntCaseLabel, part[0])
|
|
1655
|
+
expression = self._expression_direct(part)
|
|
1656
|
+
if expression is not None:
|
|
1657
|
+
label.add_child(expression)
|
|
1658
|
+
labels.add_child(label)
|
|
1659
|
+
statement = self._parse_statement()
|
|
1660
|
+
if statement is not None:
|
|
1661
|
+
labels.add_child(statement)
|
|
1662
|
+
node.add_child(labels)
|
|
1663
|
+
self._accept(";")
|
|
1664
|
+
self._ensure_progress(before)
|
|
1665
|
+
if self._accept("else") or self._accept("otherwise"):
|
|
1666
|
+
otherwise = self._node(SyntaxNodeType.ntCaseElse, self._peek(-1))
|
|
1667
|
+
self._parse_statements(otherwise, {"end"})
|
|
1668
|
+
node.add_child(otherwise)
|
|
1669
|
+
self._accept("end")
|
|
1670
|
+
return node
|
|
1671
|
+
if word == "asm":
|
|
1672
|
+
node = self._node(
|
|
1673
|
+
SyntaxNodeType.ntStatements,
|
|
1674
|
+
self._advance(),
|
|
1675
|
+
compound=True,
|
|
1676
|
+
)
|
|
1677
|
+
node.set_attribute(AttributeName.anKind, "asm")
|
|
1678
|
+
self._synchronize({"end"})
|
|
1679
|
+
self._accept("end")
|
|
1680
|
+
return node
|
|
1681
|
+
if word in _STATEMENT_TERMINATORS:
|
|
1682
|
+
return None
|
|
1683
|
+
token = self._peek()
|
|
1684
|
+
values = self._collect_until(
|
|
1685
|
+
{";", "end", "else", "until", "except", "finally"},
|
|
1686
|
+
track_angles=False,
|
|
1687
|
+
)
|
|
1688
|
+
if not values:
|
|
1689
|
+
return None
|
|
1690
|
+
assignment_index = next(
|
|
1691
|
+
(index for index, value in enumerate(values) if value.normalized == ":="),
|
|
1692
|
+
-1,
|
|
1693
|
+
)
|
|
1694
|
+
if assignment_index >= 0:
|
|
1695
|
+
node = self._node(SyntaxNodeType.ntAssign, token)
|
|
1696
|
+
lhs = self._node(SyntaxNodeType.ntLHS, token)
|
|
1697
|
+
rhs_token = values[min(assignment_index + 1, len(values) - 1)]
|
|
1698
|
+
rhs = self._node(SyntaxNodeType.ntRHS, rhs_token)
|
|
1699
|
+
left = self._expression_node(values[:assignment_index])
|
|
1700
|
+
right = self._expression_node(values[assignment_index + 1 :])
|
|
1701
|
+
if left is not None:
|
|
1702
|
+
lhs.add_child(left)
|
|
1703
|
+
if right is not None:
|
|
1704
|
+
rhs.add_child(right)
|
|
1705
|
+
node.add_child(lhs)
|
|
1706
|
+
node.add_child(rhs)
|
|
1707
|
+
return node
|
|
1708
|
+
return self._expression_direct(values)
|
|
1709
|
+
|
|
1710
|
+
def _parse_try_statement(self) -> SyntaxNode:
|
|
1711
|
+
token = self._advance()
|
|
1712
|
+
node = self._node(SyntaxNodeType.ntTry, token, compound=True)
|
|
1713
|
+
self._parse_statements(node, {"except", "finally", "end"})
|
|
1714
|
+
if self._accept("except"):
|
|
1715
|
+
except_node = self._node(SyntaxNodeType.ntExcept, self._peek(-1))
|
|
1716
|
+
while not self._eof() and not self._at("end"):
|
|
1717
|
+
before = self.index
|
|
1718
|
+
if self._accept("on"):
|
|
1719
|
+
handler = self._node(
|
|
1720
|
+
SyntaxNodeType.ntExceptionHandler,
|
|
1721
|
+
self._peek(-1),
|
|
1722
|
+
)
|
|
1723
|
+
self._collect_until(
|
|
1724
|
+
{"do", ";", "end"},
|
|
1725
|
+
track_angles=False,
|
|
1726
|
+
)
|
|
1727
|
+
self._accept("do")
|
|
1728
|
+
statement = self._parse_statement()
|
|
1729
|
+
if statement is not None:
|
|
1730
|
+
handler.add_child(statement)
|
|
1731
|
+
except_node.add_child(handler)
|
|
1732
|
+
elif self._accept("else"):
|
|
1733
|
+
statement = self._parse_statement()
|
|
1734
|
+
if statement is not None:
|
|
1735
|
+
except_node.add_child(statement)
|
|
1736
|
+
else:
|
|
1737
|
+
statement = self._parse_statement()
|
|
1738
|
+
if statement is not None:
|
|
1739
|
+
except_node.add_child(statement)
|
|
1740
|
+
self._accept(";")
|
|
1741
|
+
self._ensure_progress(before)
|
|
1742
|
+
node.add_child(except_node)
|
|
1743
|
+
elif self._accept("finally"):
|
|
1744
|
+
finally_node = self._node(SyntaxNodeType.ntFinally, self._peek(-1))
|
|
1745
|
+
self._parse_statements(finally_node, {"end"})
|
|
1746
|
+
node.add_child(finally_node)
|
|
1747
|
+
self._accept("end")
|
|
1748
|
+
return node
|
|
1749
|
+
|
|
1750
|
+
def _expression_node(self, values: list[Token]) -> SyntaxNode | None:
|
|
1751
|
+
if not values or not self.build_expressions:
|
|
1752
|
+
return None
|
|
1753
|
+
if values[0].value == "[":
|
|
1754
|
+
close = self._matching_token(values, 0, "[", "]")
|
|
1755
|
+
if close == len(values) - 1:
|
|
1756
|
+
set_node = self._node(SyntaxNodeType.ntSet, values[0])
|
|
1757
|
+
for element_tokens in self._split_top_level_tokens(
|
|
1758
|
+
values[1:close],
|
|
1759
|
+
{","},
|
|
1760
|
+
):
|
|
1761
|
+
element = self._expression_direct(element_tokens)
|
|
1762
|
+
if element is not None:
|
|
1763
|
+
set_node.add_child(element)
|
|
1764
|
+
return set_node
|
|
1765
|
+
expression = self._node(SyntaxNodeType.ntExpression, values[0])
|
|
1766
|
+
index = 0
|
|
1767
|
+
while index < len(values):
|
|
1768
|
+
token = values[index]
|
|
1769
|
+
if token.kind in {TokenKind.IDENTIFIER, TokenKind.KEYWORD}:
|
|
1770
|
+
current: SyntaxNode = self._identifier(token)
|
|
1771
|
+
index += 1
|
|
1772
|
+
while (
|
|
1773
|
+
index + 1 < len(values)
|
|
1774
|
+
and values[index].value == "."
|
|
1775
|
+
and values[index + 1].kind in {TokenKind.IDENTIFIER, TokenKind.KEYWORD}
|
|
1776
|
+
):
|
|
1777
|
+
dot = self._node(SyntaxNodeType.ntDot, values[index])
|
|
1778
|
+
dot.add_child(current)
|
|
1779
|
+
dot.add_child(self._identifier(values[index + 1]))
|
|
1780
|
+
current = dot
|
|
1781
|
+
index += 2
|
|
1782
|
+
if index < len(values) and values[index].value == "(":
|
|
1783
|
+
close = self._matching_token(values, index, "(", ")")
|
|
1784
|
+
if close is not None:
|
|
1785
|
+
call = self._node(SyntaxNodeType.ntCall, token)
|
|
1786
|
+
call.add_child(current)
|
|
1787
|
+
arguments = self._node(SyntaxNodeType.ntArguments, values[index])
|
|
1788
|
+
for argument_tokens in self._split_arguments(values[index + 1 : close]):
|
|
1789
|
+
argument_expression = self._expression_direct(argument_tokens)
|
|
1790
|
+
if argument_expression is None:
|
|
1791
|
+
continue
|
|
1792
|
+
argument = self._node(
|
|
1793
|
+
SyntaxNodeType.ntPositionalArgument,
|
|
1794
|
+
argument_tokens[0],
|
|
1795
|
+
)
|
|
1796
|
+
argument.add_child(argument_expression)
|
|
1797
|
+
arguments.add_child(argument)
|
|
1798
|
+
call.add_child(arguments)
|
|
1799
|
+
current = call
|
|
1800
|
+
index = close + 1
|
|
1801
|
+
elif index < len(values) and values[index].value == "[":
|
|
1802
|
+
close = self._matching_token(values, index, "[", "]")
|
|
1803
|
+
if close is not None:
|
|
1804
|
+
indexed = self._node(SyntaxNodeType.ntIndexed, values[index])
|
|
1805
|
+
indexed.add_child(current)
|
|
1806
|
+
indices = self._node(SyntaxNodeType.ntExpressions, values[index])
|
|
1807
|
+
for index_tokens in self._split_top_level_tokens(
|
|
1808
|
+
values[index + 1 : close],
|
|
1809
|
+
{","},
|
|
1810
|
+
):
|
|
1811
|
+
index_expression = self._expression_direct(index_tokens)
|
|
1812
|
+
if index_expression is not None:
|
|
1813
|
+
indices.add_child(index_expression)
|
|
1814
|
+
indexed.add_child(indices)
|
|
1815
|
+
current = indexed
|
|
1816
|
+
index = close + 1
|
|
1817
|
+
while index < len(values) and values[index].value == "^":
|
|
1818
|
+
deref = self._node(SyntaxNodeType.ntDeref, values[index])
|
|
1819
|
+
deref.add_child(current)
|
|
1820
|
+
current = deref
|
|
1821
|
+
index += 1
|
|
1822
|
+
expression.add_child(current)
|
|
1823
|
+
continue
|
|
1824
|
+
elif token.kind in {TokenKind.NUMBER, TokenKind.STRING, TokenKind.CHAR_CODE}:
|
|
1825
|
+
expression.add_child(self._literal(token))
|
|
1826
|
+
index += 1
|
|
1827
|
+
return expression
|
|
1828
|
+
|
|
1829
|
+
def _literal(self, token: Token) -> ValuedSyntaxNode:
|
|
1830
|
+
value = token.value
|
|
1831
|
+
if token.kind is TokenKind.STRING:
|
|
1832
|
+
value = self._dequote_string(value)
|
|
1833
|
+
literal_type = "string"
|
|
1834
|
+
elif token.kind is TokenKind.CHAR_CODE:
|
|
1835
|
+
literal_type = "char"
|
|
1836
|
+
else:
|
|
1837
|
+
literal_type = "numeric"
|
|
1838
|
+
node = self._valued(SyntaxNodeType.ntLiteral, token, value)
|
|
1839
|
+
node.set_attribute(AttributeName.anType, literal_type)
|
|
1840
|
+
return node
|
|
1841
|
+
|
|
1842
|
+
@staticmethod
|
|
1843
|
+
def _dequote_string(value: str) -> str:
|
|
1844
|
+
if value.startswith("'"):
|
|
1845
|
+
value = value[1:-1] if value.endswith("'") else value[1:]
|
|
1846
|
+
value = value.replace("''", "'")
|
|
1847
|
+
return value
|
|
1848
|
+
|
|
1849
|
+
def _identifier(self, token: Token) -> SyntaxNode:
|
|
1850
|
+
node = self._node(SyntaxNodeType.ntIdentifier, token)
|
|
1851
|
+
node.set_attribute(AttributeName.anName, self._source_token_value(token))
|
|
1852
|
+
return node
|
|
1853
|
+
|
|
1854
|
+
def _matching_token(
|
|
1855
|
+
self,
|
|
1856
|
+
values: list[Token],
|
|
1857
|
+
start: int,
|
|
1858
|
+
opening: str,
|
|
1859
|
+
closing: str,
|
|
1860
|
+
) -> int | None:
|
|
1861
|
+
depth = 0
|
|
1862
|
+
for index in range(start, len(values)):
|
|
1863
|
+
value = values[index].value
|
|
1864
|
+
if value == opening:
|
|
1865
|
+
depth += 1
|
|
1866
|
+
elif value == closing:
|
|
1867
|
+
depth -= 1
|
|
1868
|
+
if depth == 0:
|
|
1869
|
+
return index
|
|
1870
|
+
return None
|
|
1871
|
+
|
|
1872
|
+
def _top_level_token_index(
|
|
1873
|
+
self,
|
|
1874
|
+
values: list[Token],
|
|
1875
|
+
expected: str,
|
|
1876
|
+
) -> int | None:
|
|
1877
|
+
round_depth = 0
|
|
1878
|
+
square_depth = 0
|
|
1879
|
+
angle_depth = 0
|
|
1880
|
+
for index, token in enumerate(values):
|
|
1881
|
+
if (
|
|
1882
|
+
token.value == expected
|
|
1883
|
+
and round_depth == square_depth == angle_depth == 0
|
|
1884
|
+
):
|
|
1885
|
+
return index
|
|
1886
|
+
if token.value == "(":
|
|
1887
|
+
round_depth += 1
|
|
1888
|
+
elif token.value == ")":
|
|
1889
|
+
round_depth = max(0, round_depth - 1)
|
|
1890
|
+
elif token.value == "[":
|
|
1891
|
+
square_depth += 1
|
|
1892
|
+
elif token.value == "]":
|
|
1893
|
+
square_depth = max(0, square_depth - 1)
|
|
1894
|
+
elif token.value == "<":
|
|
1895
|
+
angle_depth += 1
|
|
1896
|
+
elif token.value == ">":
|
|
1897
|
+
angle_depth = max(0, angle_depth - 1)
|
|
1898
|
+
return None
|
|
1899
|
+
|
|
1900
|
+
def _split_top_level_tokens(
|
|
1901
|
+
self,
|
|
1902
|
+
values: list[Token],
|
|
1903
|
+
separators: set[str],
|
|
1904
|
+
) -> list[list[Token]]:
|
|
1905
|
+
if not values:
|
|
1906
|
+
return []
|
|
1907
|
+
result: list[list[Token]] = []
|
|
1908
|
+
current: list[Token] = []
|
|
1909
|
+
round_depth = 0
|
|
1910
|
+
square_depth = 0
|
|
1911
|
+
angle_depth = 0
|
|
1912
|
+
for token in values:
|
|
1913
|
+
if (
|
|
1914
|
+
token.value in separators
|
|
1915
|
+
and round_depth == square_depth == angle_depth == 0
|
|
1916
|
+
):
|
|
1917
|
+
result.append(current)
|
|
1918
|
+
current = []
|
|
1919
|
+
continue
|
|
1920
|
+
current.append(token)
|
|
1921
|
+
if token.value == "(":
|
|
1922
|
+
round_depth += 1
|
|
1923
|
+
elif token.value == ")":
|
|
1924
|
+
round_depth = max(0, round_depth - 1)
|
|
1925
|
+
elif token.value == "[":
|
|
1926
|
+
square_depth += 1
|
|
1927
|
+
elif token.value == "]":
|
|
1928
|
+
square_depth = max(0, square_depth - 1)
|
|
1929
|
+
elif token.value == "<":
|
|
1930
|
+
angle_depth += 1
|
|
1931
|
+
elif token.value == ">":
|
|
1932
|
+
angle_depth = max(0, angle_depth - 1)
|
|
1933
|
+
result.append(current)
|
|
1934
|
+
return result
|
|
1935
|
+
|
|
1936
|
+
def _split_arguments(self, values: list[Token]) -> list[list[Token]]:
|
|
1937
|
+
return [
|
|
1938
|
+
part
|
|
1939
|
+
for part in self._split_top_level_tokens(values, {","})
|
|
1940
|
+
if part
|
|
1941
|
+
]
|
|
1942
|
+
|
|
1943
|
+
def _read_qualified_name(self, *, allow_keywords: bool = False) -> str:
|
|
1944
|
+
if not self._is_name(allow_keywords=allow_keywords):
|
|
1945
|
+
return ""
|
|
1946
|
+
parts = [self._source_token_value(self._advance())]
|
|
1947
|
+
while self._at(".") and self._is_name(1, allow_keywords=allow_keywords):
|
|
1948
|
+
self._advance()
|
|
1949
|
+
parts.append(self._source_token_value(self._advance()))
|
|
1950
|
+
return ".".join(parts)
|
|
1951
|
+
|
|
1952
|
+
def _read_routine_name(self) -> str:
|
|
1953
|
+
if not self._is_name(allow_keywords=True):
|
|
1954
|
+
return ""
|
|
1955
|
+
parts = [self._source_token_value(self._advance())]
|
|
1956
|
+
while not self._eof():
|
|
1957
|
+
if self._at("<"):
|
|
1958
|
+
close = self._matching_stream_token("<", ">")
|
|
1959
|
+
if close is None or self._normalized_at(close + 1) != ".":
|
|
1960
|
+
break
|
|
1961
|
+
self.index = close + 1
|
|
1962
|
+
if not self._accept("."):
|
|
1963
|
+
break
|
|
1964
|
+
if not self._is_name(allow_keywords=True):
|
|
1965
|
+
break
|
|
1966
|
+
parts.append(self._source_token_value(self._advance()))
|
|
1967
|
+
return ".".join(parts)
|
|
1968
|
+
|
|
1969
|
+
def _matching_stream_token(
|
|
1970
|
+
self,
|
|
1971
|
+
opening: str,
|
|
1972
|
+
closing: str,
|
|
1973
|
+
) -> int | None:
|
|
1974
|
+
if not self._at(opening):
|
|
1975
|
+
return None
|
|
1976
|
+
depth = 0
|
|
1977
|
+
for index in range(self.index, self._last_token_index + 1):
|
|
1978
|
+
value = self.tokens[index].normalized
|
|
1979
|
+
if value == opening:
|
|
1980
|
+
depth += 1
|
|
1981
|
+
elif value == closing:
|
|
1982
|
+
depth -= 1
|
|
1983
|
+
if depth == 0:
|
|
1984
|
+
return index
|
|
1985
|
+
return None
|
|
1986
|
+
|
|
1987
|
+
def _normalized_at(self, index: int) -> str:
|
|
1988
|
+
if 0 <= index <= self._last_token_index:
|
|
1989
|
+
return self.tokens[index].normalized
|
|
1990
|
+
return ""
|
|
1991
|
+
|
|
1992
|
+
def _read_name(self) -> str:
|
|
1993
|
+
if not self._is_name(allow_keywords=True):
|
|
1994
|
+
return ""
|
|
1995
|
+
return self._source_token_value(self._advance())
|
|
1996
|
+
|
|
1997
|
+
def _collect_until(
|
|
1998
|
+
self,
|
|
1999
|
+
stops: set[str],
|
|
2000
|
+
*,
|
|
2001
|
+
max_retained: int | None = None,
|
|
2002
|
+
track_angles: bool = True,
|
|
2003
|
+
) -> list[Token]:
|
|
2004
|
+
values: list[Token] = []
|
|
2005
|
+
round_depth = 0
|
|
2006
|
+
square_depth = 0
|
|
2007
|
+
angle_depth = 0
|
|
2008
|
+
previous: Token | None = None
|
|
2009
|
+
while not self._eof():
|
|
2010
|
+
word = self._normalized()
|
|
2011
|
+
if round_depth == square_depth == angle_depth == 0 and word in stops:
|
|
2012
|
+
break
|
|
2013
|
+
token = self._advance()
|
|
2014
|
+
if max_retained is None or len(values) < max_retained:
|
|
2015
|
+
values.append(token)
|
|
2016
|
+
if token.value == "(":
|
|
2017
|
+
round_depth += 1
|
|
2018
|
+
elif token.value == ")":
|
|
2019
|
+
round_depth = max(0, round_depth - 1)
|
|
2020
|
+
elif token.value == "[":
|
|
2021
|
+
square_depth += 1
|
|
2022
|
+
elif token.value == "]":
|
|
2023
|
+
square_depth = max(0, square_depth - 1)
|
|
2024
|
+
elif (
|
|
2025
|
+
track_angles
|
|
2026
|
+
and token.value == "<"
|
|
2027
|
+
and previous is not None
|
|
2028
|
+
and previous.kind is TokenKind.IDENTIFIER
|
|
2029
|
+
and previous.end == token.start
|
|
2030
|
+
):
|
|
2031
|
+
angle_depth += 1
|
|
2032
|
+
elif track_angles and token.value == ">":
|
|
2033
|
+
angle_depth = max(0, angle_depth - 1)
|
|
2034
|
+
if token.normalized in {";", "then", "do", "begin", "end"}:
|
|
2035
|
+
angle_depth = 0
|
|
2036
|
+
previous = token
|
|
2037
|
+
return values
|
|
2038
|
+
|
|
2039
|
+
def _skip_balanced_until(self, stops: set[str]) -> None:
|
|
2040
|
+
self._collect_until(stops)
|
|
2041
|
+
|
|
2042
|
+
def _tokens_text(self, values: Iterable[Token]) -> str:
|
|
2043
|
+
text, _ = self._tokens_text_with_positions(values)
|
|
2044
|
+
return text
|
|
2045
|
+
|
|
2046
|
+
def _tokens_text_with_positions(
|
|
2047
|
+
self,
|
|
2048
|
+
values: Iterable[Token],
|
|
2049
|
+
) -> tuple[str, tuple[tuple[int, int], ...]]:
|
|
2050
|
+
result: list[str] = []
|
|
2051
|
+
positions: list[tuple[int, int]] = []
|
|
2052
|
+
previous: Token | None = None
|
|
2053
|
+
for token in values:
|
|
2054
|
+
spelling = self._source_token_value(token)
|
|
2055
|
+
if previous is not None and (
|
|
2056
|
+
previous.kind in {TokenKind.IDENTIFIER, TokenKind.KEYWORD, TokenKind.NUMBER}
|
|
2057
|
+
and token.kind in {TokenKind.IDENTIFIER, TokenKind.KEYWORD, TokenKind.NUMBER}
|
|
2058
|
+
):
|
|
2059
|
+
result.append(" ")
|
|
2060
|
+
if self.build_node_positions:
|
|
2061
|
+
positions.append((token.line, max(1, token.column - 1)))
|
|
2062
|
+
result.append(spelling)
|
|
2063
|
+
if self.build_node_positions:
|
|
2064
|
+
positions.extend(
|
|
2065
|
+
(token.line, token.column + offset)
|
|
2066
|
+
for offset in range(len(spelling))
|
|
2067
|
+
)
|
|
2068
|
+
previous = token
|
|
2069
|
+
if previous is not None and self.build_node_positions:
|
|
2070
|
+
positions.append(self._token_end_position(previous))
|
|
2071
|
+
return "".join(result), tuple(positions)
|
|
2072
|
+
|
|
2073
|
+
def _token_end_position(self, token: Token) -> tuple[int, int]:
|
|
2074
|
+
spelling = self.source[token.start:token.end]
|
|
2075
|
+
newline_count = spelling.count('\n')
|
|
2076
|
+
if not newline_count:
|
|
2077
|
+
return token.line, token.column + len(spelling)
|
|
2078
|
+
return token.line + newline_count, len(spelling.rsplit('\n', 1)[-1]) + 1
|
|
2079
|
+
|
|
2080
|
+
def _source_token_value(self, token: Token) -> str:
|
|
2081
|
+
raw_spelling = self.source[token.start:token.end]
|
|
2082
|
+
if (
|
|
2083
|
+
raw_spelling.startswith('&')
|
|
2084
|
+
and raw_spelling[1:].casefold() == token.value.casefold()
|
|
2085
|
+
):
|
|
2086
|
+
return raw_spelling
|
|
2087
|
+
return token.value
|
|
2088
|
+
|
|
2089
|
+
def _synchronize(self, stops: set[str]) -> None:
|
|
2090
|
+
while not self._eof() and self._normalized() not in stops:
|
|
2091
|
+
self._advance()
|
|
2092
|
+
|
|
2093
|
+
def _consume_through(self, value: str) -> None:
|
|
2094
|
+
self._synchronize({value})
|
|
2095
|
+
self._accept(value)
|
|
2096
|
+
|
|
2097
|
+
def _problem(self, message: str, token: Token | None = None) -> None:
|
|
2098
|
+
if len(self.problems) >= 128:
|
|
2099
|
+
return
|
|
2100
|
+
current = token or self._peek()
|
|
2101
|
+
self.problems.append(
|
|
2102
|
+
DelphiAstParseProblem(
|
|
2103
|
+
message=message[:240],
|
|
2104
|
+
line=current.line,
|
|
2105
|
+
column=current.column,
|
|
2106
|
+
)
|
|
2107
|
+
)
|
|
2108
|
+
|
|
2109
|
+
def _node(
|
|
2110
|
+
self,
|
|
2111
|
+
typ: SyntaxNodeType,
|
|
2112
|
+
token: Token,
|
|
2113
|
+
*,
|
|
2114
|
+
compound: bool = False,
|
|
2115
|
+
) -> SyntaxNode:
|
|
2116
|
+
node: SyntaxNode = CompoundSyntaxNode(typ) if compound else SyntaxNode(typ)
|
|
2117
|
+
self._set_node_position(node, token)
|
|
2118
|
+
return node
|
|
2119
|
+
|
|
2120
|
+
def _valued(self, typ: SyntaxNodeType, token: Token, value: str) -> ValuedSyntaxNode:
|
|
2121
|
+
node = ValuedSyntaxNode(typ)
|
|
2122
|
+
self._set_node_position(node, token)
|
|
2123
|
+
node.value = value
|
|
2124
|
+
return node
|
|
2125
|
+
|
|
2126
|
+
def _set_node_position(self, node: SyntaxNode, token: Token) -> None:
|
|
2127
|
+
node._file_name = self.file_name
|
|
2128
|
+
node._line = token.line
|
|
2129
|
+
node._col = token.column
|
|
2130
|
+
if self.build_node_positions:
|
|
2131
|
+
cached = self._node_position_cache.get(token.start)
|
|
2132
|
+
if cached is None:
|
|
2133
|
+
end_line, end_col = self._token_end_position(token)
|
|
2134
|
+
cached = _NodePositionAnchor(
|
|
2135
|
+
token.line,
|
|
2136
|
+
token.column,
|
|
2137
|
+
end_line,
|
|
2138
|
+
end_col,
|
|
2139
|
+
len(token.value),
|
|
2140
|
+
)
|
|
2141
|
+
self._node_position_cache[token.start] = cached
|
|
2142
|
+
node._token_end_line = cached.end_line
|
|
2143
|
+
node._token_end_col = cached.end_column
|
|
2144
|
+
node._position_anchor = cached
|
|
2145
|
+
|
|
2146
|
+
def _peek(self, offset: int = 0) -> Token:
|
|
2147
|
+
position = self.index + offset
|
|
2148
|
+
if position < 0:
|
|
2149
|
+
position = 0
|
|
2150
|
+
elif position > self._last_token_index:
|
|
2151
|
+
position = self._last_token_index
|
|
2152
|
+
return self.tokens[position]
|
|
2153
|
+
|
|
2154
|
+
def _normalized(self, offset: int = 0) -> str:
|
|
2155
|
+
return self._peek(offset).normalized
|
|
2156
|
+
|
|
2157
|
+
def _at(self, value: str, offset: int = 0) -> bool:
|
|
2158
|
+
return self._normalized(offset) == value
|
|
2159
|
+
|
|
2160
|
+
def _accept(self, value: str) -> bool:
|
|
2161
|
+
if not self._at(value):
|
|
2162
|
+
return False
|
|
2163
|
+
self._advance()
|
|
2164
|
+
return True
|
|
2165
|
+
|
|
2166
|
+
def _advance(self) -> Token:
|
|
2167
|
+
token = self._peek()
|
|
2168
|
+
if token.kind is not TokenKind.EOF:
|
|
2169
|
+
self.index += 1
|
|
2170
|
+
return token
|
|
2171
|
+
|
|
2172
|
+
def _eof(self) -> bool:
|
|
2173
|
+
return self._peek().kind is TokenKind.EOF
|
|
2174
|
+
|
|
2175
|
+
def _is_name(self, offset: int = 0, *, allow_keywords: bool = False) -> bool:
|
|
2176
|
+
kind = self._peek(offset).kind
|
|
2177
|
+
return kind is TokenKind.IDENTIFIER or (allow_keywords and kind is TokenKind.KEYWORD)
|
|
2178
|
+
|
|
2179
|
+
def _ensure_progress(self, before: int) -> None:
|
|
2180
|
+
if self.index <= before and not self._eof():
|
|
2181
|
+
self._problem("Parser recovery forced forward progress")
|
|
2182
|
+
self._advance()
|
|
2183
|
+
|
|
2184
|
+
|
|
2185
|
+
__all__ = [
|
|
2186
|
+
"DelphiAstParseOutput",
|
|
2187
|
+
"DelphiAstParseProblem",
|
|
2188
|
+
"DelphiAstParser",
|
|
2189
|
+
]
|