timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
tcs/treesitter_packer.py
ADDED
|
@@ -0,0 +1,1446 @@
|
|
|
1
|
+
"""Phase 1 of the resilient parser: a Tree-sitter CST -> CPython `ast` translator, standalone.
|
|
2
|
+
|
|
3
|
+
Nothing in the engine imports this module yet. `ast_scanner.py` does not know it exists, so the
|
|
4
|
+
current scanner's behaviour, findings and speed are untouched by construction.
|
|
5
|
+
|
|
6
|
+
Why Tree-sitter: its grammar parses *incomplete and future* Python (PEP 695 generics, PEP 701
|
|
7
|
+
nested f-strings, a half-typed function) and recovers with `ERROR` nodes instead of raising, so a
|
|
8
|
+
file no longer vanishes because of one line. This module turns that concrete syntax tree back into
|
|
9
|
+
ordinary `ast.AST` nodes, which means every existing consumer - `ast.walk`, `ast.NodeVisitor`, the
|
|
10
|
+
scanner's own collectors - can read it with no changes.
|
|
11
|
+
|
|
12
|
+
Three properties, in order of importance:
|
|
13
|
+
|
|
14
|
+
1. **Never panics.** `parse_to_ast` returns a valid, traversable `ast.Module` for *any* input,
|
|
15
|
+
including garbage bytes, and is wrapped so that no exception can escape it. An unmodellable
|
|
16
|
+
statement becomes `ast.Pass` at its own coordinates; an unmodellable expression becomes a
|
|
17
|
+
`ast.Constant(None)` placeholder (or, if it has translatable children, those children are
|
|
18
|
+
re-hosted in an `ast.Tuple` so their calls and attributes stay reachable to `ast.walk`).
|
|
19
|
+
2. **Exact coordinates.** Every node gets `lineno` / `col_offset` / `end_lineno` / `end_col_offset`
|
|
20
|
+
from the CST span (Tree-sitter counts lines from 0, `ast` from 1), so a finding reported on a
|
|
21
|
+
recovered file points at the same line it would have pointed at in a natively parsed file.
|
|
22
|
+
3. **Bounded memory.** The `Tree` is created inside one call and is a local of that call only - it
|
|
23
|
+
is never stored on the module, in a cache, or on a class, so its CST is freed as soon as the
|
|
24
|
+
conversion returns (the 0.26 Python binding has no `Tree.destroy()`; refcount release is the
|
|
25
|
+
mechanism, and `destroy()` is called first when a newer binding does expose it). Only the grammar
|
|
26
|
+
and one `Parser` are cached, and the parser is guarded by a lock because its internal scratch
|
|
27
|
+
buffer is not reentrant.
|
|
28
|
+
|
|
29
|
+
Scope of the node set: module, function/class definitions (sync, async, decorated, PEP 695
|
|
30
|
+
generics), assignments (plain, chained, annotated, augmented), expression statements, return,
|
|
31
|
+
import/from, if/elif/else, for/while (+ else), try/except/else/finally, with, break/continue/pass/
|
|
32
|
+
raise/global/nonlocal/assert/delete, PEP 695 `type X = ...`; and expressions: calls (positional,
|
|
33
|
+
keyword, `*args`, `**kw`, generator-argument), attribute chains, subscripts and slices, names with
|
|
34
|
+
correct Load/Store/Del context, literals, implicit-concatenated strings, f-strings as `JoinedStr`
|
|
35
|
+
with the interpolated expressions preserved as real nodes, list/dict/set, comprehensions (list, set,
|
|
36
|
+
dict, generator), lambda, walrus, await, yield/yield from, starred, conditional, and
|
|
37
|
+
binary/unary/boolean/comparison operators. Anything still outside that set is degraded to a
|
|
38
|
+
placeholder, never to an exception.
|
|
39
|
+
"""
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import ast
|
|
43
|
+
import threading
|
|
44
|
+
from typing import Any, List, Optional, Sequence, Tuple
|
|
45
|
+
|
|
46
|
+
__all__ = ["parse_to_ast", "clean_parse", "tree_sitter_available", "SYNTAX_PLACEHOLDER_PREFIX"]
|
|
47
|
+
|
|
48
|
+
# Markers used when a fragment could not be modelled. They are strings in the AST only to make the
|
|
49
|
+
# degradation visible in a dump; the node types themselves are plain `ast` nodes.
|
|
50
|
+
SYNTAX_PLACEHOLDER_PREFIX = "<unsupported:"
|
|
51
|
+
|
|
52
|
+
_LANG: Any = None
|
|
53
|
+
_PARSER: Any = None
|
|
54
|
+
_PARSER_LOCK = threading.Lock()
|
|
55
|
+
_LOAD = ast.Load
|
|
56
|
+
_STORE = ast.Store
|
|
57
|
+
_DEL = ast.Del
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _Unavailable(Exception):
|
|
61
|
+
"""Tree-sitter is not installed / the grammar did not load."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _points(node) -> Tuple[int, int, int, int]:
|
|
65
|
+
"""(lineno, col_offset, end_lineno, end_col_offset) of a CST node.
|
|
66
|
+
|
|
67
|
+
Tree-sitter names these `start_point`/`end_point` in some releases and `start_position`/
|
|
68
|
+
`end_position` in others; both are tried so the module keeps working across versions.
|
|
69
|
+
"""
|
|
70
|
+
for start_attr, end_attr in (("start_point", "end_point"),
|
|
71
|
+
("start_position", "end_position")):
|
|
72
|
+
getter = getattr(node, start_attr, None)
|
|
73
|
+
if getter is not None:
|
|
74
|
+
start, end = tuple(getter), tuple(getattr(node, end_attr))
|
|
75
|
+
return start[0] + 1, start[1], end[0] + 1, end[1]
|
|
76
|
+
return 1, 0, 1, 0
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _text(node) -> str:
|
|
80
|
+
raw = getattr(node, "text", None) or b""
|
|
81
|
+
try:
|
|
82
|
+
return raw.decode("utf-8", "replace")
|
|
83
|
+
except Exception:
|
|
84
|
+
return ""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def tree_sitter_available() -> bool:
|
|
88
|
+
"""True when the grammar loads. Callers that need certainty check this, not the parser."""
|
|
89
|
+
try:
|
|
90
|
+
_parser()
|
|
91
|
+
return True
|
|
92
|
+
except Exception:
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _parser():
|
|
97
|
+
"""One cached Parser over one cached Language; never holds a tree."""
|
|
98
|
+
global _LANG, _PARSER
|
|
99
|
+
if _PARSER is not None:
|
|
100
|
+
return _PARSER
|
|
101
|
+
try:
|
|
102
|
+
import tree_sitter_python
|
|
103
|
+
from tree_sitter import Language, Parser
|
|
104
|
+
except Exception as exc: # ImportError, or a broken wheel
|
|
105
|
+
raise _Unavailable(f"tree-sitter unavailable: {exc}") from exc
|
|
106
|
+
language = getattr(tree_sitter_python, "language", None)
|
|
107
|
+
if language is None:
|
|
108
|
+
raise _Unavailable("tree_sitter_python.language is missing")
|
|
109
|
+
cap = language()
|
|
110
|
+
_LANG = cap if isinstance(cap, Language) else Language(cap)
|
|
111
|
+
try:
|
|
112
|
+
_PARSER = Parser(_LANG)
|
|
113
|
+
except TypeError:
|
|
114
|
+
p = Parser()
|
|
115
|
+
p.set_language(_LANG)
|
|
116
|
+
_PARSER = p
|
|
117
|
+
return _PARSER
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _pos(node, out: ast.AST) -> ast.AST:
|
|
121
|
+
"""Stamp CST coordinates onto an `ast` node. Called for every node that has a span."""
|
|
122
|
+
if node is None or out is None:
|
|
123
|
+
return out
|
|
124
|
+
lineno, col, end_lineno, end_col = _points(node)
|
|
125
|
+
out.lineno = lineno
|
|
126
|
+
out.col_offset = col
|
|
127
|
+
out.end_lineno = end_lineno
|
|
128
|
+
out.end_col_offset = end_col
|
|
129
|
+
return out
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _encode(source_code) -> bytes:
|
|
133
|
+
if isinstance(source_code, (bytes, bytearray)):
|
|
134
|
+
return bytes(source_code)
|
|
135
|
+
if isinstance(source_code, str):
|
|
136
|
+
return source_code.encode("utf-8", "surrogatepass")
|
|
137
|
+
return str(source_code).encode("utf-8", "replace")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def parse_to_ast(source_code, filename: str = "<unknown>") -> ast.Module:
|
|
141
|
+
"""Parse `source_code` into a traversable `ast.Module`. Never raises."""
|
|
142
|
+
data = _encode(source_code)
|
|
143
|
+
try:
|
|
144
|
+
parser = None
|
|
145
|
+
try:
|
|
146
|
+
with _PARSER_LOCK:
|
|
147
|
+
parser = _parser()
|
|
148
|
+
tree = parser.parse(data)
|
|
149
|
+
try:
|
|
150
|
+
module = _Translator(source=source_code, filename=filename).module(tree.root_node)
|
|
151
|
+
finally:
|
|
152
|
+
# The CST is dropped here: only the converted `ast` survives this call, so no
|
|
153
|
+
# tree-sitter memory is retained once the function returns.
|
|
154
|
+
destroy = getattr(tree, "destroy", None)
|
|
155
|
+
if callable(destroy):
|
|
156
|
+
destroy()
|
|
157
|
+
del tree
|
|
158
|
+
finally:
|
|
159
|
+
parser = None
|
|
160
|
+
module.filename = filename if isinstance(filename, str) else "<unknown>"
|
|
161
|
+
return module
|
|
162
|
+
except Exception:
|
|
163
|
+
# Zero-panic contract: a caller must always get something `ast.walk` can traverse.
|
|
164
|
+
fallback = ast.Module(body=[ast.Pass()], type_ignores=[])
|
|
165
|
+
ast.fix_missing_locations(fallback)
|
|
166
|
+
return fallback
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def clean_parse(source_code, filename: str = "<treesitter>") -> Optional[ast.Module]:
|
|
170
|
+
"""Return a module only for source the grammar reads as *complete, valid* Python, else None.
|
|
171
|
+
|
|
172
|
+
This is the gate a scanner needs before it trusts a recovered tree. `parse_to_ast` always answers,
|
|
173
|
+
because its contract is "never panic"; that same eagerness makes it unsafe as an unconditional
|
|
174
|
+
rescue, since `def f(:` also yields a module - one whose broken statement was guessed at. Three
|
|
175
|
+
independent proofs have to hold here instead, and all three are facts about the input, never
|
|
176
|
+
hand-picked name lists:
|
|
177
|
+
|
|
178
|
+
1. the grammar's own verdict - `root.has_error` is False, so no `ERROR` node was needed to read
|
|
179
|
+
the file; this is what separates "valid syntax this interpreter predates" (PEP 695/701) from
|
|
180
|
+
"the author's file is damaged";
|
|
181
|
+
2. the translator's own verdict - it never had to synthesise a statement (`_pass`) for something
|
|
182
|
+
it could not model, so no code was quietly dropped or invented;
|
|
183
|
+
3. CPython's own verdict - the finished module compiles, which rejects recovered trees that are
|
|
184
|
+
structurally walkable but not valid Python (a hoisted top-level `return`, a starred target).
|
|
185
|
+
|
|
186
|
+
A file that fails any proof returns None, and the caller keeps its existing, proven recovery.
|
|
187
|
+
Never raises.
|
|
188
|
+
"""
|
|
189
|
+
try:
|
|
190
|
+
with _PARSER_LOCK:
|
|
191
|
+
parser = _parser()
|
|
192
|
+
tree = parser.parse(_encode(source_code))
|
|
193
|
+
try:
|
|
194
|
+
root = tree.root_node
|
|
195
|
+
if root.has_error:
|
|
196
|
+
return None
|
|
197
|
+
translator = _Translator(source=source_code, filename=filename)
|
|
198
|
+
module = translator.module(root)
|
|
199
|
+
finally:
|
|
200
|
+
destroy = getattr(tree, "destroy", None)
|
|
201
|
+
if callable(destroy):
|
|
202
|
+
destroy()
|
|
203
|
+
del tree
|
|
204
|
+
if translator.degraded:
|
|
205
|
+
return None
|
|
206
|
+
name = filename if isinstance(filename, str) else "<treesitter>"
|
|
207
|
+
module.filename = name
|
|
208
|
+
compile(module, name, "exec")
|
|
209
|
+
return module
|
|
210
|
+
except Exception:
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class _Translator:
|
|
215
|
+
"""CST -> `ast`. Every method returns `ast` nodes or placeholders, never None, never raises."""
|
|
216
|
+
|
|
217
|
+
def __init__(self, source, filename: str):
|
|
218
|
+
self.source = source
|
|
219
|
+
self.filename = filename
|
|
220
|
+
# Set whenever a statement had to be invented instead of translated. `clean_parse` reads it
|
|
221
|
+
# to decide whether the tree it built may be handed to a scanner.
|
|
222
|
+
self.degraded = False
|
|
223
|
+
|
|
224
|
+
# ─── module ───
|
|
225
|
+
def module(self, root) -> ast.Module:
|
|
226
|
+
body = self._statements(root)
|
|
227
|
+
if not body:
|
|
228
|
+
body = [ast.Pass()]
|
|
229
|
+
module = ast.Module(body=body, type_ignores=[])
|
|
230
|
+
_pos(root, module)
|
|
231
|
+
ast.fix_missing_locations(module)
|
|
232
|
+
return module
|
|
233
|
+
|
|
234
|
+
def _block(self, node) -> List[ast.AST]:
|
|
235
|
+
"""A `block` node (or any node used as a suite) -> statement list, never empty."""
|
|
236
|
+
if node is None:
|
|
237
|
+
return [self._pass(None, 1, 0)]
|
|
238
|
+
stmts = self._statements(node)
|
|
239
|
+
return stmts or [self._pass(node)]
|
|
240
|
+
|
|
241
|
+
def _statements(self, container) -> List[ast.AST]:
|
|
242
|
+
out: List[ast.AST] = []
|
|
243
|
+
for child in self._named(container):
|
|
244
|
+
node = self._statement(child)
|
|
245
|
+
if node is None:
|
|
246
|
+
continue
|
|
247
|
+
if isinstance(node, list):
|
|
248
|
+
out.extend(node)
|
|
249
|
+
else:
|
|
250
|
+
out.append(node)
|
|
251
|
+
return out
|
|
252
|
+
|
|
253
|
+
def _named(self, node) -> List[Any]:
|
|
254
|
+
try:
|
|
255
|
+
return [c for c in node.named_children if c is not None and not c.is_missing
|
|
256
|
+
and c.type not in ("comment",)]
|
|
257
|
+
except Exception:
|
|
258
|
+
return []
|
|
259
|
+
|
|
260
|
+
def _kids(self, node) -> List[Any]:
|
|
261
|
+
try:
|
|
262
|
+
return [c for c in node.children if c is not None and not c.is_missing]
|
|
263
|
+
except Exception:
|
|
264
|
+
return []
|
|
265
|
+
|
|
266
|
+
def _pass(self, node, lineno: int = 1, col: int = 0) -> ast.Pass:
|
|
267
|
+
# Every call site of this helper is "something real was here and we could not model it"; a
|
|
268
|
+
# `pass` written in the source goes through the `pass_statement` handler instead.
|
|
269
|
+
self.degraded = True
|
|
270
|
+
out = ast.Pass()
|
|
271
|
+
if node is None:
|
|
272
|
+
out.lineno, out.col_offset = lineno, col
|
|
273
|
+
out.end_lineno, out.end_col_offset = lineno, col + 4
|
|
274
|
+
else:
|
|
275
|
+
_pos(node, out)
|
|
276
|
+
return out
|
|
277
|
+
|
|
278
|
+
# ─── statements ───
|
|
279
|
+
def _statement(self, node):
|
|
280
|
+
handler = self._STMT.get(node.type)
|
|
281
|
+
if handler is not None:
|
|
282
|
+
try:
|
|
283
|
+
made = handler(self, node)
|
|
284
|
+
except Exception:
|
|
285
|
+
# A handler that choked on a damaged node still must not cost the file its code:
|
|
286
|
+
# salvage whatever statements are recognisable inside it.
|
|
287
|
+
return self._error(node)
|
|
288
|
+
self._absorb_errors(node, made)
|
|
289
|
+
return made
|
|
290
|
+
if node.type == "ERROR":
|
|
291
|
+
return self._error(node)
|
|
292
|
+
if node.is_missing:
|
|
293
|
+
return None
|
|
294
|
+
# Anything statement-shaped we do not model still has to keep its place in the block.
|
|
295
|
+
return self._pass(node)
|
|
296
|
+
|
|
297
|
+
def _absorb_errors(self, node, made) -> None:
|
|
298
|
+
"""Keep code that tree-sitter parked *beside* a recovered body.
|
|
299
|
+
|
|
300
|
+
For `class C:\\n def m(): ...\\n bad(:` the grammar puts the damaged line in an ERROR node
|
|
301
|
+
that is a sibling of the class body block, so the intact method would vanish with it. The
|
|
302
|
+
statements inside that sibling are moved into the recovered statement's own body. Clean
|
|
303
|
+
source has no ERROR nodes, so this never touches well-formed code or its ordering.
|
|
304
|
+
"""
|
|
305
|
+
targets = [made] if isinstance(made, ast.AST) else list(made or [])
|
|
306
|
+
bodies = []
|
|
307
|
+
for target in targets:
|
|
308
|
+
body = getattr(target, "body", None)
|
|
309
|
+
if isinstance(body, list) and body and all(isinstance(stmt, ast.stmt) for stmt in body):
|
|
310
|
+
bodies.append(body)
|
|
311
|
+
if not bodies:
|
|
312
|
+
return
|
|
313
|
+
extra: List[ast.AST] = []
|
|
314
|
+
for child in self._named(node):
|
|
315
|
+
if child.type == "ERROR":
|
|
316
|
+
extra.extend(self._salvage(child))
|
|
317
|
+
if not extra:
|
|
318
|
+
return
|
|
319
|
+
for body in bodies:
|
|
320
|
+
body.extend(extra)
|
|
321
|
+
# Source order has to survive: the salvaged span usually precedes the body block.
|
|
322
|
+
body.sort(key=lambda stmt: (getattr(stmt, "lineno", 0), getattr(stmt, "col_offset", 0)))
|
|
323
|
+
|
|
324
|
+
def _salvage(self, node) -> List[ast.AST]:
|
|
325
|
+
"""Recover the recognisable statements inside a broken region; may legitimately be empty."""
|
|
326
|
+
salvaged: List[ast.AST] = []
|
|
327
|
+
for child in self._named(node):
|
|
328
|
+
if child.type == "ERROR":
|
|
329
|
+
salvaged.extend(self._salvage(child))
|
|
330
|
+
continue
|
|
331
|
+
handler = self._STMT.get(child.type)
|
|
332
|
+
if handler is None or child.type in ("module", "block"):
|
|
333
|
+
# A wrapper (block, argument list, ...) is not a statement of its own: look inside.
|
|
334
|
+
salvaged.extend(self._salvage(child))
|
|
335
|
+
continue
|
|
336
|
+
try:
|
|
337
|
+
# Route through `_statement`, not the bare handler, so a container recovered here
|
|
338
|
+
# gets the same beside-body salvage as one recovered from a clean block.
|
|
339
|
+
made = self._statement(child)
|
|
340
|
+
except Exception:
|
|
341
|
+
salvaged.extend(self._salvage(child))
|
|
342
|
+
continue
|
|
343
|
+
if made is None:
|
|
344
|
+
continue
|
|
345
|
+
salvaged.extend(made if isinstance(made, list) else [made])
|
|
346
|
+
return salvaged
|
|
347
|
+
|
|
348
|
+
def _error(self, node):
|
|
349
|
+
# A fully lost region still has to leave a statement behind so the block stays valid.
|
|
350
|
+
return self._salvage(node) or [self._pass(node)]
|
|
351
|
+
|
|
352
|
+
def _stmt_module(self, root):
|
|
353
|
+
return self._statements(root) or [self._pass(root)]
|
|
354
|
+
|
|
355
|
+
def _expr_statement(self, node):
|
|
356
|
+
inner = self._named(node)
|
|
357
|
+
if not inner:
|
|
358
|
+
return self._pass(node)
|
|
359
|
+
child = inner[0]
|
|
360
|
+
if child.type in ("assignment", "augmented_assignment"):
|
|
361
|
+
return self._assignment(child)
|
|
362
|
+
expr = self._expression(child, _LOAD)
|
|
363
|
+
return _pos(node, ast.Expr(value=expr))
|
|
364
|
+
|
|
365
|
+
def _assignment(self, node):
|
|
366
|
+
left = node.child_by_field_name("left")
|
|
367
|
+
right = node.child_by_field_name("right")
|
|
368
|
+
annotation = node.child_by_field_name("type")
|
|
369
|
+
operator = self._operator_token(node)
|
|
370
|
+
if operator in _AUGOP and left is not None and right is not None:
|
|
371
|
+
return _pos(node, ast.AugAssign(target=self._expression(left, _STORE),
|
|
372
|
+
op=_AUGOP[operator](),
|
|
373
|
+
value=self._expression(right, _LOAD)))
|
|
374
|
+
if annotation is not None and left is not None:
|
|
375
|
+
target = self._expression(left, _STORE)
|
|
376
|
+
if isinstance(target, (ast.Name, ast.Attribute, ast.Subscript)):
|
|
377
|
+
return _pos(node, ast.AnnAssign(
|
|
378
|
+
target=target, annotation=self._unwrap_type(annotation),
|
|
379
|
+
value=self._expression(right, _LOAD) if right is not None else None,
|
|
380
|
+
simple=1))
|
|
381
|
+
if left is None or right is None:
|
|
382
|
+
return self._pass(node)
|
|
383
|
+
# `a = b = value` nests: tree-sitter puts the rest of the chain in `right`.
|
|
384
|
+
targets = [self._target(left)]
|
|
385
|
+
tail = right
|
|
386
|
+
while tail is not None and tail.type == "assignment":
|
|
387
|
+
nxt_left = tail.child_by_field_name("left")
|
|
388
|
+
nxt_right = tail.child_by_field_name("right")
|
|
389
|
+
if nxt_left is None or nxt_right is None:
|
|
390
|
+
break
|
|
391
|
+
targets.append(self._target(nxt_left))
|
|
392
|
+
tail = nxt_right
|
|
393
|
+
value = self._expression(tail, _LOAD) if tail is not None else _pos(node, ast.Constant(None))
|
|
394
|
+
return _pos(node, ast.Assign(targets=targets, value=value))
|
|
395
|
+
|
|
396
|
+
def _target(self, node):
|
|
397
|
+
return self._expression(node, _STORE)
|
|
398
|
+
|
|
399
|
+
def _function(self, node, decorators=None):
|
|
400
|
+
name_node = node.child_by_field_name("name")
|
|
401
|
+
params_node = node.child_by_field_name("parameters")
|
|
402
|
+
body_node = node.child_by_field_name("body")
|
|
403
|
+
returns = node.child_by_field_name("return_type")
|
|
404
|
+
args = self._parameters(params_node)
|
|
405
|
+
decorator_list = decorators or []
|
|
406
|
+
body = self._block(body_node)
|
|
407
|
+
name = _text(name_node).strip() or "_unnamed"
|
|
408
|
+
returns_ast = self._unwrap_type(returns) if returns is not None else None
|
|
409
|
+
is_async = any(c.type == "async" for c in self._kids(node))
|
|
410
|
+
kind = ast.AsyncFunctionDef if is_async else ast.FunctionDef
|
|
411
|
+
out = kind(name=name, args=args, body=body, decorator_list=decorator_list,
|
|
412
|
+
returns=returns_ast)
|
|
413
|
+
if "type_params" in getattr(kind, "_attributes", ()):
|
|
414
|
+
# PEP 695 type parameters are dropped (the grammar keeps them on the definition);
|
|
415
|
+
# setting the field keeps the node constructible on 3.12+ consumers.
|
|
416
|
+
out.type_params = []
|
|
417
|
+
_pos(node, out)
|
|
418
|
+
return out
|
|
419
|
+
|
|
420
|
+
def _class(self, node, decorators=None):
|
|
421
|
+
name_node = node.child_by_field_name("name")
|
|
422
|
+
supers = node.child_by_field_name("superclasses")
|
|
423
|
+
body_node = node.child_by_field_name("body")
|
|
424
|
+
bases: List[ast.AST] = []
|
|
425
|
+
keywords: List[ast.keyword] = []
|
|
426
|
+
if supers is not None:
|
|
427
|
+
for child in self._named(supers):
|
|
428
|
+
if child.type == "keyword_argument":
|
|
429
|
+
keywords.append(self._keyword(child))
|
|
430
|
+
continue
|
|
431
|
+
bases.append(self._expression(child, _LOAD))
|
|
432
|
+
out = ast.ClassDef(name=_text(name_node).strip() or "_unnamed",
|
|
433
|
+
bases=bases, keywords=keywords,
|
|
434
|
+
body=self._block(body_node),
|
|
435
|
+
decorator_list=decorators or [])
|
|
436
|
+
if "type_params" in getattr(ast.ClassDef, "_attributes", ()):
|
|
437
|
+
out.type_params = []
|
|
438
|
+
return _pos(node, out)
|
|
439
|
+
|
|
440
|
+
def _decorated(self, node):
|
|
441
|
+
decorators: List[ast.AST] = []
|
|
442
|
+
inner = None
|
|
443
|
+
for child in self._named(node):
|
|
444
|
+
if child.type == "decorator":
|
|
445
|
+
target = self._named(child)
|
|
446
|
+
decorators.append(self._expression(target[-1], _LOAD) if target
|
|
447
|
+
else _pos(child, ast.Constant(None)))
|
|
448
|
+
elif child.type in self._STMT:
|
|
449
|
+
inner = child
|
|
450
|
+
if inner is None:
|
|
451
|
+
return [self._pass(node)] + decorators
|
|
452
|
+
made = self._STMT[inner.type](self, inner, decorators) if inner.type in (
|
|
453
|
+
"function_definition", "class_definition") else self._statement(inner)
|
|
454
|
+
return made if isinstance(made, list) else [made]
|
|
455
|
+
|
|
456
|
+
def _return(self, node):
|
|
457
|
+
value = self._named(node)
|
|
458
|
+
return _pos(node, ast.Return(self._expression(value[0], _LOAD) if value else None))
|
|
459
|
+
|
|
460
|
+
def _imported_names(self, node) -> List[ast.alias]:
|
|
461
|
+
"""Every named child in a name position: the module itself is not an imported alias."""
|
|
462
|
+
names: List[ast.alias] = []
|
|
463
|
+
for index, child in enumerate(self._kids(node)):
|
|
464
|
+
if not child.is_named or child.is_missing:
|
|
465
|
+
continue
|
|
466
|
+
if self._field(node, index) == "module_name":
|
|
467
|
+
continue
|
|
468
|
+
names.append(self._alias(child))
|
|
469
|
+
return names or [ast.alias(name="_", asname=None)]
|
|
470
|
+
|
|
471
|
+
def _import(self, node):
|
|
472
|
+
return _pos(node, ast.Import(names=self._imported_names(node)))
|
|
473
|
+
|
|
474
|
+
def _future_import(self, node):
|
|
475
|
+
return _pos(node, ast.ImportFrom(module="__future__",
|
|
476
|
+
names=self._imported_names(node), level=0))
|
|
477
|
+
|
|
478
|
+
def _import_from(self, node):
|
|
479
|
+
module_node = node.child_by_field_name("module_name")
|
|
480
|
+
names = self._imported_names(node)
|
|
481
|
+
level, module = 0, None
|
|
482
|
+
if module_node is not None:
|
|
483
|
+
if module_node.type == "relative_import":
|
|
484
|
+
prefix = module_node.child_by_field_name("import_prefix") or next(
|
|
485
|
+
(c for c in self._kids(module_node) if c.type == "import_prefix"), None)
|
|
486
|
+
level = len(_text(prefix).strip()) if prefix is not None else 1
|
|
487
|
+
dotted = next((c for c in self._named(module_node) if c.type == "dotted_name"), None)
|
|
488
|
+
module = _text(dotted) if dotted is not None else None
|
|
489
|
+
else:
|
|
490
|
+
module = _text(module_node)
|
|
491
|
+
return _pos(node, ast.ImportFrom(module=module, names=names, level=level))
|
|
492
|
+
|
|
493
|
+
def _alias(self, child) -> ast.alias:
|
|
494
|
+
if child.type == "aliased_import":
|
|
495
|
+
real = child.child_by_field_name("name")
|
|
496
|
+
alias = child.child_by_field_name("alias")
|
|
497
|
+
name = _text(real).strip() or _text(child).split(" as ")[0].strip()
|
|
498
|
+
return _pos(child, ast.alias(name=name, asname=_text(alias).strip() or None))
|
|
499
|
+
if child.type == "wildcard_import":
|
|
500
|
+
return _pos(child, ast.alias(name="*", asname=None))
|
|
501
|
+
return _pos(child, ast.alias(name=_text(child).strip(), asname=None))
|
|
502
|
+
|
|
503
|
+
def _if(self, node):
|
|
504
|
+
condition = node.child_by_field_name("condition")
|
|
505
|
+
consequence = node.child_by_field_name("consequence")
|
|
506
|
+
alternatives = [c for c in self._named(node)
|
|
507
|
+
if c.type in ("elif_clause", "else_clause")]
|
|
508
|
+
out = ast.If(test=self._expression(condition, _LOAD) if condition is not None
|
|
509
|
+
else _pos(node, ast.Constant(None)),
|
|
510
|
+
body=self._block(consequence), orelse=[])
|
|
511
|
+
tail = out
|
|
512
|
+
for alt in alternatives:
|
|
513
|
+
if alt.type == "elif_clause":
|
|
514
|
+
nested = ast.If(
|
|
515
|
+
test=self._expression(alt.child_by_field_name("condition"), _LOAD),
|
|
516
|
+
body=self._block(alt.child_by_field_name("consequence")), orelse=[])
|
|
517
|
+
_pos(alt, nested)
|
|
518
|
+
tail.orelse = [nested]
|
|
519
|
+
tail = nested
|
|
520
|
+
else:
|
|
521
|
+
tail.orelse = self._block(alt.child_by_field_name("body")
|
|
522
|
+
or alt.child_by_field_name("consequence"))
|
|
523
|
+
return _pos(node, out)
|
|
524
|
+
|
|
525
|
+
def _for(self, node):
|
|
526
|
+
# `async for` is a different node class in `ast`, not a flag on `ast.For`.
|
|
527
|
+
kind = ast.AsyncFor if any(c.type == "async" for c in self._kids(node)) else ast.For
|
|
528
|
+
return _pos(node, kind(
|
|
529
|
+
target=self._expression(node.child_by_field_name("left"), _STORE),
|
|
530
|
+
iter=self._expression(node.child_by_field_name("right"), _LOAD),
|
|
531
|
+
body=self._block(node.child_by_field_name("body")),
|
|
532
|
+
orelse=self._else_of(node)))
|
|
533
|
+
|
|
534
|
+
def _while(self, node):
|
|
535
|
+
return _pos(node, ast.While(test=self._expression(node.child_by_field_name("condition"), _LOAD),
|
|
536
|
+
body=self._block(node.child_by_field_name("body")),
|
|
537
|
+
orelse=self._else_of(node)))
|
|
538
|
+
|
|
539
|
+
def _else_of(self, node) -> List[ast.AST]:
|
|
540
|
+
alt = next((c for c in self._named(node) if c.type == "else_clause"), None)
|
|
541
|
+
if alt is None:
|
|
542
|
+
return []
|
|
543
|
+
body = alt.child_by_field_name("body") or next(
|
|
544
|
+
(c for c in self._named(alt) if c.type == "block"), None)
|
|
545
|
+
return self._block(body)
|
|
546
|
+
|
|
547
|
+
def _try(self, node):
|
|
548
|
+
body = []
|
|
549
|
+
handlers: List[ast.ExceptHandler] = []
|
|
550
|
+
orelse: List[ast.AST] = []
|
|
551
|
+
finalbody: List[ast.AST] = []
|
|
552
|
+
star = False
|
|
553
|
+
for child in self._named(node):
|
|
554
|
+
if child.type == "block" and not body:
|
|
555
|
+
body = self._block(child)
|
|
556
|
+
elif child.type == "except_clause":
|
|
557
|
+
handlers.append(self._except(child))
|
|
558
|
+
# `except*` is a different node class (`ast.TryStar`), decided by that `*`.
|
|
559
|
+
star = star or any(c.type == "*" for c in self._kids(child))
|
|
560
|
+
elif child.type == "else_clause":
|
|
561
|
+
orelse = self._else_of(child) or self._block(child.child_by_field_name("body"))
|
|
562
|
+
elif child.type == "finally_clause":
|
|
563
|
+
suite = child.child_by_field_name("body") or next(
|
|
564
|
+
(c for c in self._named(child) if c.type == "block"), None)
|
|
565
|
+
finalbody = self._block(suite)
|
|
566
|
+
kind = ast.TryStar if star and hasattr(ast, "TryStar") else ast.Try
|
|
567
|
+
return _pos(node, kind(body=body or [self._pass(node)], handlers=handlers,
|
|
568
|
+
orelse=orelse, finalbody=finalbody))
|
|
569
|
+
|
|
570
|
+
def _except(self, node) -> ast.ExceptHandler:
|
|
571
|
+
value = node.child_by_field_name("value")
|
|
572
|
+
exc = value
|
|
573
|
+
alias = node.child_by_field_name("alias")
|
|
574
|
+
if value is not None and value.type == "as_pattern":
|
|
575
|
+
parts = self._named(value)
|
|
576
|
+
exc = parts[0] if parts else None
|
|
577
|
+
alias = value.child_by_field_name("alias")
|
|
578
|
+
name = (_text(alias).strip() or None) if alias is not None else None
|
|
579
|
+
block = next((c for c in self._named(node) if c.type == "block"), None)
|
|
580
|
+
return _pos(node, ast.ExceptHandler(type=self._expression(exc, _LOAD) if exc is not None
|
|
581
|
+
else None, name=name, body=self._block(block)))
|
|
582
|
+
|
|
583
|
+
def _with(self, node):
|
|
584
|
+
items: List[ast.withitem] = []
|
|
585
|
+
for child in self._named(node):
|
|
586
|
+
if child.type == "with_clause":
|
|
587
|
+
for item in self._named(child):
|
|
588
|
+
items.append(self._with_item(item))
|
|
589
|
+
elif child.type in ("with_item", "as_pattern"):
|
|
590
|
+
items.append(self._with_item(child))
|
|
591
|
+
if not items:
|
|
592
|
+
items = [ast.withitem(context_expr=_pos(node, ast.Constant(None)),
|
|
593
|
+
optional_vars=None)]
|
|
594
|
+
# `async with` is its own class, not a flag on `ast.With`.
|
|
595
|
+
kind = ast.AsyncWith if any(c.type == "async" for c in self._kids(node)) else ast.With
|
|
596
|
+
return _pos(node, kind(items=items, body=self._block(node.child_by_field_name("body"))))
|
|
597
|
+
|
|
598
|
+
def _with_item(self, node) -> ast.withitem:
|
|
599
|
+
value = node.child_by_field_name("value") or (
|
|
600
|
+
self._named(node)[0] if self._named(node) else None)
|
|
601
|
+
if value is not None and value.type == "as_pattern":
|
|
602
|
+
parts = self._named(value)
|
|
603
|
+
context = parts[0] if parts else value
|
|
604
|
+
alias = value.child_by_field_name("alias")
|
|
605
|
+
return _pos(node, ast.withitem(context_expr=self._expression(context, _LOAD),
|
|
606
|
+
optional_vars=self._expression(alias, _STORE)
|
|
607
|
+
if alias is not None else None))
|
|
608
|
+
return _pos(node, ast.withitem(
|
|
609
|
+
context_expr=self._expression(value, _LOAD) if value is not None
|
|
610
|
+
else _pos(node, ast.Constant(None)), optional_vars=None))
|
|
611
|
+
|
|
612
|
+
def _break(self, node):
|
|
613
|
+
return _pos(node, ast.Break())
|
|
614
|
+
|
|
615
|
+
def _continue(self, node):
|
|
616
|
+
return _pos(node, ast.Continue())
|
|
617
|
+
|
|
618
|
+
def _pass_stmt(self, node):
|
|
619
|
+
return self._pass(node)
|
|
620
|
+
|
|
621
|
+
def _raise(self, node):
|
|
622
|
+
cause_node = node.child_by_field_name("cause")
|
|
623
|
+
parts = [c for c in self._named(node) if c is not cause_node]
|
|
624
|
+
exc = self._expression(parts[0], _LOAD) if parts else None
|
|
625
|
+
cause = self._expression(cause_node, _LOAD) if cause_node is not None else None
|
|
626
|
+
return _pos(node, ast.Raise(exc=exc, cause=cause))
|
|
627
|
+
|
|
628
|
+
def _global(self, node, nonlocal_=False):
|
|
629
|
+
names = [_text(c).strip() for c in self._named(node) if c.type == "identifier"]
|
|
630
|
+
kind = ast.Nonlocal if nonlocal_ else ast.Global
|
|
631
|
+
return _pos(node, kind(names=names or ["_"]))
|
|
632
|
+
|
|
633
|
+
def _assert(self, node):
|
|
634
|
+
parts = self._named(node)
|
|
635
|
+
test = self._expression(parts[0], _LOAD) if parts else _pos(node, ast.Constant(None))
|
|
636
|
+
msg = self._expression(parts[1], _LOAD) if len(parts) > 1 else None
|
|
637
|
+
return _pos(node, ast.Assert(test=test, msg=msg))
|
|
638
|
+
|
|
639
|
+
def _delete(self, node):
|
|
640
|
+
parts = self._named(node)
|
|
641
|
+
if len(parts) == 1 and parts[0].type == "expression_list":
|
|
642
|
+
parts = self._named(parts[0])
|
|
643
|
+
targets = [self._expression(c, _DEL) for c in parts]
|
|
644
|
+
if not targets:
|
|
645
|
+
targets = [self._pass(node)]
|
|
646
|
+
return _pos(node, ast.Delete(targets=targets))
|
|
647
|
+
|
|
648
|
+
def _type_alias(self, node):
|
|
649
|
+
# PEP 695 `type X = int`: modelled as `X = int` so downstream name tracking still sees it.
|
|
650
|
+
left = node.child_by_field_name("left")
|
|
651
|
+
right = node.child_by_field_name("right")
|
|
652
|
+
if left is None:
|
|
653
|
+
return self._pass(node)
|
|
654
|
+
target = self._expression(left, _STORE)
|
|
655
|
+
while isinstance(target, ast.Subscript):
|
|
656
|
+
# `type Alias[T] = ...`: the tracked name is `Alias`, the `[T]` is a type parameter.
|
|
657
|
+
target = target.value
|
|
658
|
+
if isinstance(target, (ast.Name, ast.Attribute)):
|
|
659
|
+
target.ctx = _STORE()
|
|
660
|
+
value = self._unwrap_type(right) if right is not None else _pos(node, ast.Constant(None))
|
|
661
|
+
return _pos(node, ast.Assign(targets=[target], value=value))
|
|
662
|
+
|
|
663
|
+
# ─── match (PEP 634) ───
|
|
664
|
+
def _match(self, node):
|
|
665
|
+
if not hasattr(ast, "Match"):
|
|
666
|
+
return self._pass(node)
|
|
667
|
+
subject = node.child_by_field_name("subject")
|
|
668
|
+
cases: List[ast.AST] = []
|
|
669
|
+
for child in self._named(node):
|
|
670
|
+
if child.type == "case_clause":
|
|
671
|
+
cases.append(self._case_clause(child))
|
|
672
|
+
elif child.type == "block":
|
|
673
|
+
for inner in self._named(child):
|
|
674
|
+
if inner.type == "case_clause":
|
|
675
|
+
cases.append(self._case_clause(inner))
|
|
676
|
+
return _pos(node, ast.Match(subject=self._expression(subject, _LOAD), cases=cases))
|
|
677
|
+
|
|
678
|
+
def _case_clause(self, node) -> ast.AST:
|
|
679
|
+
pattern = None
|
|
680
|
+
guard = None
|
|
681
|
+
for child in self._named(node):
|
|
682
|
+
if child.type == "case_pattern" and pattern is None:
|
|
683
|
+
pattern = self._pattern(child)
|
|
684
|
+
elif child.type == "if_clause":
|
|
685
|
+
condition = child.child_by_field_name("condition") or self._param_name(child)
|
|
686
|
+
guard = self._expression(condition, _LOAD)
|
|
687
|
+
body = node.child_by_field_name("consequence") or node.child_by_field_name("body")
|
|
688
|
+
return _pos(node, ast.match_case(pattern=pattern or ast.MatchAs(),
|
|
689
|
+
guard=guard, body=self._block(body)))
|
|
690
|
+
|
|
691
|
+
def _pattern(self, node) -> ast.AST:
|
|
692
|
+
if node is None:
|
|
693
|
+
return ast.MatchAs()
|
|
694
|
+
handler = self._PATTERN.get(node.type)
|
|
695
|
+
if handler is not None:
|
|
696
|
+
try:
|
|
697
|
+
return handler(self, node)
|
|
698
|
+
except Exception:
|
|
699
|
+
pass
|
|
700
|
+
return ast.MatchAs()
|
|
701
|
+
|
|
702
|
+
def _field(self, node, index):
|
|
703
|
+
try:
|
|
704
|
+
return node.field_name_for_child(index)
|
|
705
|
+
except Exception:
|
|
706
|
+
return None
|
|
707
|
+
|
|
708
|
+
def _pattern_unwrap(self, node) -> ast.AST:
|
|
709
|
+
"""case_pattern / group_pattern are wrappers: translate what is inside them."""
|
|
710
|
+
inner = self._named(node)
|
|
711
|
+
return self._pattern(inner[0]) if inner else ast.MatchAs()
|
|
712
|
+
|
|
713
|
+
def _pattern_value(self, node) -> ast.AST:
|
|
714
|
+
# A one-word `dotted_name` in a pattern is a capture (`case direction`), and `_` is the
|
|
715
|
+
# wildcard; only a real `a.b` chain is a value comparison.
|
|
716
|
+
if node.type == "dotted_name":
|
|
717
|
+
words = [c for c in self._kids(node) if c.type == "identifier"]
|
|
718
|
+
if len(words) == 1:
|
|
719
|
+
name = _text(words[0]).strip()
|
|
720
|
+
return _pos(node, ast.MatchAs(name=None if name == "_" else name))
|
|
721
|
+
return _pos(node, ast.MatchValue(value=self._expression(node, _LOAD)))
|
|
722
|
+
|
|
723
|
+
def _pattern_sequence(self, node) -> ast.AST:
|
|
724
|
+
return _pos(node, ast.MatchSequence(
|
|
725
|
+
patterns=[self._pattern(c) for c in self._named(node)]))
|
|
726
|
+
|
|
727
|
+
def _pattern_star(self, node) -> ast.AST:
|
|
728
|
+
"""`case [a, *rest]`: the star is a MatchStar inside MatchSequence.patterns."""
|
|
729
|
+
inner = self._last_named(node)
|
|
730
|
+
name = _text(inner if inner is not None else node).strip().lstrip("*").strip()
|
|
731
|
+
return _pos(node, ast.MatchStar(name=name or None))
|
|
732
|
+
|
|
733
|
+
def _pattern_or(self, node) -> ast.AST:
|
|
734
|
+
return _pos(node, ast.MatchOr(
|
|
735
|
+
patterns=[self._pattern(c) for c in self._named(node)]))
|
|
736
|
+
|
|
737
|
+
def _pattern_as(self, node) -> ast.AST:
|
|
738
|
+
parts = self._named(node)
|
|
739
|
+
alias = node.child_by_field_name("alias")
|
|
740
|
+
inner = parts[0] if parts else None
|
|
741
|
+
if alias is None and len(parts) > 1:
|
|
742
|
+
alias = parts[-1]
|
|
743
|
+
name = _text(alias).strip() if alias is not None else ""
|
|
744
|
+
return _pos(node, ast.MatchAs(pattern=self._pattern(inner), name=name or None))
|
|
745
|
+
|
|
746
|
+
def _pattern_mapping(self, node) -> ast.AST:
|
|
747
|
+
keys: List[ast.AST] = []
|
|
748
|
+
patterns: List[ast.AST] = []
|
|
749
|
+
rest = None
|
|
750
|
+
pending: Optional[ast.AST] = None
|
|
751
|
+
for index, child in enumerate(self._kids(node)):
|
|
752
|
+
field = self._field(node, index)
|
|
753
|
+
if field == "key":
|
|
754
|
+
# MatchMapping keys are bare literals (`Constant`), not MatchValue wrappers.
|
|
755
|
+
pending = self._expression(child, _LOAD)
|
|
756
|
+
elif field == "value":
|
|
757
|
+
keys.append(pending if pending is not None else _pos(child, ast.Constant(None)))
|
|
758
|
+
patterns.append(self._pattern(child))
|
|
759
|
+
pending = None
|
|
760
|
+
elif child.type in ("splat_pattern", "double_starred_pattern"):
|
|
761
|
+
inner = self._last_named(child)
|
|
762
|
+
rest = (_text(inner if inner is not None else child).strip()
|
|
763
|
+
.lstrip("*").strip() or None)
|
|
764
|
+
# The value slot of MatchMapping is named `patterns`, not `values`.
|
|
765
|
+
return _pos(node, ast.MatchMapping(keys=keys, patterns=patterns, rest=rest))
|
|
766
|
+
|
|
767
|
+
def _pattern_class(self, node) -> ast.AST:
|
|
768
|
+
flat: List[Any] = []
|
|
769
|
+
for child in self._named(node):
|
|
770
|
+
if child.type == "case_pattern":
|
|
771
|
+
inner = self._named(child)
|
|
772
|
+
flat.append(inner[0] if inner else child)
|
|
773
|
+
else:
|
|
774
|
+
flat.append(child)
|
|
775
|
+
patterns: List[ast.AST] = []
|
|
776
|
+
kwd_attrs: List[str] = []
|
|
777
|
+
kwd_patterns: List[ast.AST] = []
|
|
778
|
+
for child in flat[1:]:
|
|
779
|
+
if child.type != "keyword_pattern":
|
|
780
|
+
patterns.append(self._pattern(child))
|
|
781
|
+
continue
|
|
782
|
+
key = (child.child_by_field_name("name") or child.child_by_field_name("key")
|
|
783
|
+
or self._param_name(child))
|
|
784
|
+
value = (child.child_by_field_name("pattern") or child.child_by_field_name("value")
|
|
785
|
+
or self._last_named(child))
|
|
786
|
+
kwd_attrs.append(_text(key).strip())
|
|
787
|
+
kwd_patterns.append(self._pattern(value))
|
|
788
|
+
cls = flat[0] if flat else None
|
|
789
|
+
# MatchClass keyword arguments are two parallel lists (`kwd_attrs` / `kwd_patterns`).
|
|
790
|
+
return _pos(node, ast.MatchClass(cls=self._expression(cls, _LOAD), patterns=patterns,
|
|
791
|
+
kwd_attrs=kwd_attrs, kwd_patterns=kwd_patterns))
|
|
792
|
+
|
|
793
|
+
def _pattern_capture(self, node) -> ast.AST:
|
|
794
|
+
return _pos(node, ast.MatchAs(name=_text(node).strip() or None))
|
|
795
|
+
|
|
796
|
+
def _pattern_wildcard(self, node) -> ast.AST:
|
|
797
|
+
return _pos(node, ast.MatchAs())
|
|
798
|
+
|
|
799
|
+
# ─── parameters ───
|
|
800
|
+
def _parameters(self, node) -> ast.arguments:
|
|
801
|
+
"""`ast.arguments` with the buckets CPython uses.
|
|
802
|
+
|
|
803
|
+
A parameter is positional-only only when the source actually writes `/`; the marker's
|
|
804
|
+
position, not the parameter's, decides the bucket. Defaults are kept aligned to the tail of
|
|
805
|
+
`posonlyargs + args` and keyword defaults to `kwonlyargs`, exactly as `ast.parse` does.
|
|
806
|
+
"""
|
|
807
|
+
if node is None:
|
|
808
|
+
return ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[],
|
|
809
|
+
kw_defaults=[], kwarg=None, defaults=[])
|
|
810
|
+
positional: List[Tuple[ast.arg, Optional[ast.AST]]] = []
|
|
811
|
+
pos_only_count = 0
|
|
812
|
+
kwonly: List[ast.arg] = []
|
|
813
|
+
kw_defaults: List[Optional[ast.AST]] = []
|
|
814
|
+
after_star = False
|
|
815
|
+
vararg = None
|
|
816
|
+
varkw = None
|
|
817
|
+
for child in self._kids(node):
|
|
818
|
+
kind = child.type
|
|
819
|
+
if kind in ("(", ")", "comment"):
|
|
820
|
+
continue
|
|
821
|
+
if kind == "positional_separator":
|
|
822
|
+
pos_only_count = len(positional)
|
|
823
|
+
continue
|
|
824
|
+
if kind == "keyword_separator":
|
|
825
|
+
after_star = True
|
|
826
|
+
continue
|
|
827
|
+
if kind == "list_splat_pattern":
|
|
828
|
+
target = child.child_by_field_name("argument") or self._last_named(child)
|
|
829
|
+
if target is not None:
|
|
830
|
+
vararg = self._arg(target)
|
|
831
|
+
after_star = True
|
|
832
|
+
continue
|
|
833
|
+
if kind == "dictionary_splat_pattern":
|
|
834
|
+
target = child.child_by_field_name("pattern") or self._last_named(child)
|
|
835
|
+
if target is not None:
|
|
836
|
+
varkw = self._arg(target)
|
|
837
|
+
continue
|
|
838
|
+
default = child.child_by_field_name("value")
|
|
839
|
+
default_expr = self._expression(default, _LOAD) if default is not None else None
|
|
840
|
+
if kind in ("typed_default_parameter", "default_parameter", "typed_parameter",
|
|
841
|
+
"identifier"):
|
|
842
|
+
made = self._arg(child)
|
|
843
|
+
if after_star:
|
|
844
|
+
kwonly.append(made)
|
|
845
|
+
kw_defaults.append(default_expr)
|
|
846
|
+
else:
|
|
847
|
+
positional.append((made, default_expr))
|
|
848
|
+
continue
|
|
849
|
+
# Anything exotic (PEP 695 type parameters, patterns): keep it out of the signature
|
|
850
|
+
# rather than crash on it.
|
|
851
|
+
only_pos, rest = positional[:pos_only_count], positional[pos_only_count:]
|
|
852
|
+
return ast.arguments(
|
|
853
|
+
posonlyargs=[arg for arg, _ in only_pos],
|
|
854
|
+
args=[arg for arg, _ in rest],
|
|
855
|
+
vararg=vararg,
|
|
856
|
+
kwonlyargs=kwonly,
|
|
857
|
+
kw_defaults=kw_defaults,
|
|
858
|
+
kwarg=varkw,
|
|
859
|
+
defaults=[default for _, default in only_pos + rest if default is not None])
|
|
860
|
+
|
|
861
|
+
def _last_named(self, node):
|
|
862
|
+
named = self._named(node)
|
|
863
|
+
return named[-1] if named else None
|
|
864
|
+
|
|
865
|
+
def _param_name(self, node):
|
|
866
|
+
"""The identifier inside a parameter node, wherever the grammar put it."""
|
|
867
|
+
for field in ("name", "pattern", "argument"):
|
|
868
|
+
found = node.child_by_field_name(field)
|
|
869
|
+
if found is not None:
|
|
870
|
+
return found
|
|
871
|
+
named = self._named(node)
|
|
872
|
+
return named[0] if named else node
|
|
873
|
+
|
|
874
|
+
def _arg(self, node) -> ast.arg:
|
|
875
|
+
annotation = None
|
|
876
|
+
name_node = node
|
|
877
|
+
if node.type != "identifier":
|
|
878
|
+
name_node = self._param_name(node)
|
|
879
|
+
ann = node.child_by_field_name("type")
|
|
880
|
+
annotation = self._unwrap_type(ann) if ann is not None else None
|
|
881
|
+
name = _text(name_node).strip() or "_"
|
|
882
|
+
out = ast.arg(arg=name, annotation=annotation)
|
|
883
|
+
_pos(node, out)
|
|
884
|
+
return out
|
|
885
|
+
|
|
886
|
+
# ─── expressions ───
|
|
887
|
+
def _unwrap_type(self, node):
|
|
888
|
+
"""tree-sitter wraps annotations in a `type` node; the value lives one level down."""
|
|
889
|
+
if node is None:
|
|
890
|
+
return None
|
|
891
|
+
if node.type == "type":
|
|
892
|
+
inner = self._named(node)
|
|
893
|
+
if len(inner) == 1:
|
|
894
|
+
return self._expression(inner[0], _LOAD)
|
|
895
|
+
if not inner:
|
|
896
|
+
return _pos(node, ast.Constant(None))
|
|
897
|
+
return self._generic_expression(node, _LOAD)
|
|
898
|
+
return self._expression(node, _LOAD)
|
|
899
|
+
|
|
900
|
+
def _expression(self, node, ctx=_LOAD):
|
|
901
|
+
if node is None:
|
|
902
|
+
return _pos(None, ast.Constant(None))
|
|
903
|
+
try:
|
|
904
|
+
handler = self._EXPR.get(node.type)
|
|
905
|
+
if handler is not None:
|
|
906
|
+
return handler(self, node, ctx)
|
|
907
|
+
if node.type == "ERROR":
|
|
908
|
+
return self._placeholder(node)
|
|
909
|
+
if node.type == "type":
|
|
910
|
+
return self._unwrap_type(node) or self._placeholder(node)
|
|
911
|
+
return self._generic_expression(node, ctx)
|
|
912
|
+
except Exception:
|
|
913
|
+
return self._placeholder(node)
|
|
914
|
+
|
|
915
|
+
def _generic_expression(self, node, ctx):
|
|
916
|
+
"""Unknown construct: re-host whatever inside it we can translate."""
|
|
917
|
+
kept: List[ast.AST] = []
|
|
918
|
+
for child in self._named(node):
|
|
919
|
+
if child.type in ("comment", "string_content", "string_start", "string_end"):
|
|
920
|
+
continue
|
|
921
|
+
kept.append(self._expression(child, ctx))
|
|
922
|
+
if len(kept) == 1:
|
|
923
|
+
# Keep the inner node's own coordinates: the wrapper span would shift `**kw` style
|
|
924
|
+
# offsets away from where CPython reports the name.
|
|
925
|
+
return kept[0]
|
|
926
|
+
if kept:
|
|
927
|
+
return _pos(node, ast.Tuple(elts=kept, ctx=_LOAD()))
|
|
928
|
+
return self._placeholder(node)
|
|
929
|
+
|
|
930
|
+
def _placeholder(self, node):
|
|
931
|
+
# An expression we could not model is a hole in the tree, not a translated node.
|
|
932
|
+
self.degraded = True
|
|
933
|
+
out = ast.Constant(value=None)
|
|
934
|
+
_pos(node, out)
|
|
935
|
+
return out
|
|
936
|
+
|
|
937
|
+
def _name(self, node, ctx):
|
|
938
|
+
return _pos(node, ast.Name(id=_text(node).strip() or "_", ctx=ctx()))
|
|
939
|
+
|
|
940
|
+
def _dotted(self, node, ctx=_LOAD):
|
|
941
|
+
"""`a.b.c` becomes the same Attribute chain CPython builds (only the outer node takes ctx)."""
|
|
942
|
+
parts = [c for c in self._kids(node) if c.type == "identifier"]
|
|
943
|
+
if not parts:
|
|
944
|
+
names = [p for p in _text(node).strip().split(".") if p]
|
|
945
|
+
if not names:
|
|
946
|
+
return self._placeholder(node)
|
|
947
|
+
out = _pos(node, ast.Name(id=names[0], ctx=_LOAD()))
|
|
948
|
+
for name in names[1:]:
|
|
949
|
+
out = ast.Attribute(value=out, attr=name, ctx=_LOAD())
|
|
950
|
+
return _pos(node, out)
|
|
951
|
+
out = _pos(parts[0], ast.Name(id=_text(parts[0]).strip(), ctx=_LOAD()))
|
|
952
|
+
for child in parts[1:]:
|
|
953
|
+
out = ast.Attribute(value=out, attr=_text(child).strip(), ctx=_LOAD())
|
|
954
|
+
if isinstance(out, (ast.Name, ast.Attribute)):
|
|
955
|
+
out.ctx = ctx()
|
|
956
|
+
return _pos(node, out)
|
|
957
|
+
|
|
958
|
+
def _attribute(self, node, ctx):
|
|
959
|
+
value = node.child_by_field_name("object")
|
|
960
|
+
attr = node.child_by_field_name("attribute")
|
|
961
|
+
if value is None:
|
|
962
|
+
parts = [c for c in self._kids(node) if c.type != "."]
|
|
963
|
+
value, attr = (parts[0], parts[-1]) if len(parts) > 1 else (None, None)
|
|
964
|
+
inner = self._expression(value, _LOAD) if value is not None else _pos(node, ast.Constant(None))
|
|
965
|
+
out = ast.Attribute(value=inner, attr=(_text(attr).strip() or "_"), ctx=ctx())
|
|
966
|
+
return _pos(node, out)
|
|
967
|
+
|
|
968
|
+
def _subscript(self, node, ctx):
|
|
969
|
+
value = node.child_by_field_name("value")
|
|
970
|
+
sub = node.child_by_field_name("subscript")
|
|
971
|
+
if sub is None:
|
|
972
|
+
sub = next((c for c in self._named(node) if c.type != "slice"), None)
|
|
973
|
+
slice_node = self._slice(sub) if sub is not None and sub.type == "slice" \
|
|
974
|
+
else self._expression(sub, _LOAD)
|
|
975
|
+
inner = self._expression(value, _LOAD) if value is not None else _pos(node, ast.Constant(None))
|
|
976
|
+
return _pos(node, ast.Subscript(value=inner, slice=slice_node, ctx=ctx()))
|
|
977
|
+
|
|
978
|
+
def _slice(self, node):
|
|
979
|
+
slots: List[Optional[ast.AST]] = [None, None, None]
|
|
980
|
+
slot = 0
|
|
981
|
+
saw_colon = False
|
|
982
|
+
for child in self._kids(node):
|
|
983
|
+
if child.type == ":":
|
|
984
|
+
saw_colon = True
|
|
985
|
+
slot = min(slot + 1, 2)
|
|
986
|
+
elif child.is_named and not child.is_missing:
|
|
987
|
+
slots[slot] = self._expression(child, _LOAD)
|
|
988
|
+
if not saw_colon:
|
|
989
|
+
named = self._named(node)
|
|
990
|
+
return self._expression(named[0], _LOAD) if named else self._placeholder(node)
|
|
991
|
+
lower, upper, step = slots
|
|
992
|
+
return _pos(node, ast.Slice(lower=lower, upper=upper, step=step))
|
|
993
|
+
|
|
994
|
+
def _call(self, node, ctx):
|
|
995
|
+
func = node.child_by_field_name("function")
|
|
996
|
+
arguments = node.child_by_field_name("arguments")
|
|
997
|
+
args: List[ast.AST] = []
|
|
998
|
+
keywords: List[ast.keyword] = []
|
|
999
|
+
# `sum(x for x in rows)`: the grammar makes the parentheses a generator_expression node,
|
|
1000
|
+
# and CPython models that single argument as a positional GeneratorExp.
|
|
1001
|
+
if arguments is not None and arguments.type == "generator_expression":
|
|
1002
|
+
args = [self._comprehension(arguments, _LOAD, ast.GeneratorExp)]
|
|
1003
|
+
for child in self._named(arguments) if (arguments is not None
|
|
1004
|
+
and arguments.type != "generator_expression") else []:
|
|
1005
|
+
if child.type == "keyword_argument":
|
|
1006
|
+
keywords.append(self._keyword(child))
|
|
1007
|
+
elif child.type in ("list_splat", "list_splat_pattern", "starred_expression"):
|
|
1008
|
+
args.append(_pos(child, ast.Starred(
|
|
1009
|
+
value=self._expression(self._param_name(child), _LOAD), ctx=_LOAD())))
|
|
1010
|
+
elif child.type in ("dictionary_splat", "dictionary_splat_pattern"):
|
|
1011
|
+
keywords.append(_pos(child, ast.keyword(
|
|
1012
|
+
arg=None, value=self._expression(self._param_name(child), _LOAD))))
|
|
1013
|
+
else:
|
|
1014
|
+
args.append(self._expression(child, _LOAD))
|
|
1015
|
+
callee = self._expression(func, _LOAD) if func is not None else _pos(node, ast.Name(
|
|
1016
|
+
id="_unnamed_call", ctx=_LOAD()))
|
|
1017
|
+
return _pos(node, ast.Call(func=callee, args=args, keywords=keywords))
|
|
1018
|
+
|
|
1019
|
+
def _keyword(self, node) -> ast.keyword:
|
|
1020
|
+
key = node.child_by_field_name("name")
|
|
1021
|
+
value = node.child_by_field_name("value")
|
|
1022
|
+
return _pos(node, ast.keyword(arg=_text(key).strip() or None,
|
|
1023
|
+
value=self._expression(value, _LOAD)))
|
|
1024
|
+
|
|
1025
|
+
def _operator_token(self, node):
|
|
1026
|
+
for child in self._kids(node):
|
|
1027
|
+
if not child.is_named and child.type not in ("(", ")", "[", "]", "{", "}", ","):
|
|
1028
|
+
return child.type
|
|
1029
|
+
return ""
|
|
1030
|
+
|
|
1031
|
+
def _binary(self, node, ctx):
|
|
1032
|
+
left = node.child_by_field_name("left")
|
|
1033
|
+
right = node.child_by_field_name("right")
|
|
1034
|
+
op = self._operator_token(node)
|
|
1035
|
+
return _pos(node, ast.BinOp(left=self._expression(left, _LOAD),
|
|
1036
|
+
op=_BINOP.get(op, ast.Add)(),
|
|
1037
|
+
right=self._expression(right, _LOAD)))
|
|
1038
|
+
|
|
1039
|
+
def _boolean(self, node, ctx):
|
|
1040
|
+
left = node.child_by_field_name("left")
|
|
1041
|
+
right = node.child_by_field_name("right")
|
|
1042
|
+
op = _BOOLOP.get(self._operator_token(node), ast.Or)()
|
|
1043
|
+
values = [self._expression(left, _LOAD), self._expression(right, _LOAD)]
|
|
1044
|
+
# CPython flattens `a or b or c` into one BoolOp; tree-sitter nests it, so re-flatten.
|
|
1045
|
+
first, last = values[0], values[-1]
|
|
1046
|
+
if isinstance(first, ast.BoolOp) and type(first.op) is type(op):
|
|
1047
|
+
values = list(first.values) + values[1:]
|
|
1048
|
+
elif isinstance(last, ast.BoolOp) and type(last.op) is type(op):
|
|
1049
|
+
values = values[:-1] + list(last.values)
|
|
1050
|
+
return _pos(node, ast.BoolOp(op=op, values=values))
|
|
1051
|
+
|
|
1052
|
+
def _comparison(self, node, ctx):
|
|
1053
|
+
ops: List[ast.cmpop] = []
|
|
1054
|
+
parts: List[ast.AST] = []
|
|
1055
|
+
for child in self._kids(node):
|
|
1056
|
+
if child.is_named:
|
|
1057
|
+
parts.append(self._expression(child, _LOAD))
|
|
1058
|
+
elif child.type in _CMPOP:
|
|
1059
|
+
ops.append(_CMPOP[child.type]())
|
|
1060
|
+
if not parts:
|
|
1061
|
+
return self._placeholder(node)
|
|
1062
|
+
left = parts[0]
|
|
1063
|
+
comparators = parts[1:]
|
|
1064
|
+
if len(ops) != len(comparators):
|
|
1065
|
+
ops = (ops + [ast.NotEq() for _ in comparators])[:max(0, len(comparators))]
|
|
1066
|
+
return _pos(node, ast.Compare(left=left, ops=ops, comparators=comparators))
|
|
1067
|
+
|
|
1068
|
+
def _unary(self, node, ctx):
|
|
1069
|
+
argument = node.child_by_field_name("argument")
|
|
1070
|
+
if argument is None:
|
|
1071
|
+
named = self._named(node)
|
|
1072
|
+
argument = named[-1] if named else None
|
|
1073
|
+
op_type = node.type
|
|
1074
|
+
if op_type == "not_operator":
|
|
1075
|
+
op = ast.Not()
|
|
1076
|
+
else:
|
|
1077
|
+
op = _UNOP.get(self._operator_token(node), ast.USub)()
|
|
1078
|
+
return _pos(node, ast.UnaryOp(op=op, operand=self._expression(argument, _LOAD)))
|
|
1079
|
+
|
|
1080
|
+
def _constant(self, node, ctx):
|
|
1081
|
+
try:
|
|
1082
|
+
value = ast.literal_eval(_text(node).strip())
|
|
1083
|
+
except Exception:
|
|
1084
|
+
return self._placeholder(node)
|
|
1085
|
+
return _pos(node, ast.Constant(value=value))
|
|
1086
|
+
|
|
1087
|
+
def _string(self, node, ctx):
|
|
1088
|
+
text = _text(node)
|
|
1089
|
+
if node.type == "concatenated_string":
|
|
1090
|
+
return self._concatenated(node, text)
|
|
1091
|
+
parts = self._named(node)
|
|
1092
|
+
if not any(c.type == "interpolation" for c in parts):
|
|
1093
|
+
try:
|
|
1094
|
+
return _pos(node, ast.Constant(value=ast.literal_eval(text)))
|
|
1095
|
+
except Exception:
|
|
1096
|
+
# An unterminated string still has to yield a node with the right span.
|
|
1097
|
+
return _pos(node, ast.Constant(value=text))
|
|
1098
|
+
values: List[ast.AST] = []
|
|
1099
|
+
for child in parts:
|
|
1100
|
+
if child.type == "interpolation":
|
|
1101
|
+
values.append(self._interpolation(child))
|
|
1102
|
+
elif child.type == "string_content" and _text(child):
|
|
1103
|
+
values.append(_pos(child, ast.Constant(value=_text(child))))
|
|
1104
|
+
return _pos(node, ast.JoinedStr(values=values or [_pos(node, ast.Constant(value=text))]))
|
|
1105
|
+
|
|
1106
|
+
def _concatenated(self, node, text):
|
|
1107
|
+
"""`"a" f"b{x}" 'c'`: CPython merges adjacent literals, so the values are merged too."""
|
|
1108
|
+
values: List[ast.AST] = []
|
|
1109
|
+
constants: List[ast.AST] = []
|
|
1110
|
+
for child in self._named(node):
|
|
1111
|
+
made = self._expression(child, _LOAD)
|
|
1112
|
+
if isinstance(made, ast.JoinedStr):
|
|
1113
|
+
values.extend(made.values)
|
|
1114
|
+
else:
|
|
1115
|
+
values.append(made)
|
|
1116
|
+
if isinstance(made, ast.Constant):
|
|
1117
|
+
constants.append(made)
|
|
1118
|
+
if constants and len(constants) == len(values):
|
|
1119
|
+
if all(isinstance(c.value, (bytes, bytearray)) for c in constants):
|
|
1120
|
+
return _pos(node, ast.Constant(value=b"".join(bytes(c.value) for c in constants)))
|
|
1121
|
+
if all(isinstance(c.value, str) for c in constants):
|
|
1122
|
+
return _pos(node, ast.Constant(value="".join(c.value for c in constants)))
|
|
1123
|
+
return _pos(node, ast.JoinedStr(values=values or [_pos(node, ast.Constant(value=text))]))
|
|
1124
|
+
|
|
1125
|
+
def _interpolation(self, node) -> ast.FormattedValue:
|
|
1126
|
+
# Node objects are re-created per access, so identity cannot mark "the expression child";
|
|
1127
|
+
# the field name at the child index can.
|
|
1128
|
+
expr_index = -1
|
|
1129
|
+
kids = self._kids(node)
|
|
1130
|
+
for index, child in enumerate(kids):
|
|
1131
|
+
if self._field(node, index) == "expression":
|
|
1132
|
+
expr_index = index
|
|
1133
|
+
break
|
|
1134
|
+
expr = kids[expr_index] if expr_index >= 0 else self._first_named(node)
|
|
1135
|
+
conversion = -1
|
|
1136
|
+
format_spec = None
|
|
1137
|
+
for index, child in enumerate(kids):
|
|
1138
|
+
if index == expr_index or child.type in ("{", "}", "!", ":"):
|
|
1139
|
+
continue
|
|
1140
|
+
text = _text(child).strip()
|
|
1141
|
+
if text.startswith("!") and len(text) == 2:
|
|
1142
|
+
conversion = ord(text[1])
|
|
1143
|
+
elif text.startswith(":"):
|
|
1144
|
+
format_spec = self._format_spec(child, text)
|
|
1145
|
+
return _pos(node, ast.FormattedValue(value=self._expression(expr, _LOAD),
|
|
1146
|
+
conversion=conversion, format_spec=format_spec))
|
|
1147
|
+
|
|
1148
|
+
def _format_spec(self, node, raw: str) -> ast.AST:
|
|
1149
|
+
"""CPython models every format spec as a JoinedStr: nested fields (`{x:{w}}`) become real
|
|
1150
|
+
FormattedValues and the spec text around them becomes literals. tree-sitter reports only the
|
|
1151
|
+
fields as children, so each literal run is the gap between field spans inside the spec text.
|
|
1152
|
+
Reading the gaps in order is what keeps `:>{pad}` a Constant-then-FormattedValue, the way
|
|
1153
|
+
CPython builds it, instead of lumping all the spec text in front."""
|
|
1154
|
+
text = raw if isinstance(raw, str) else ""
|
|
1155
|
+
# The grammar's spec node spans the leading `:`; CPython's `format_spec` does not include it.
|
|
1156
|
+
lead = 1 if text.startswith(":") else 0
|
|
1157
|
+
body = text[lead:]
|
|
1158
|
+
base = self._byte_base(node)
|
|
1159
|
+
values: List[ast.AST] = []
|
|
1160
|
+
cursor = 0
|
|
1161
|
+
for child in self._named(node):
|
|
1162
|
+
if child.type not in ("format_expression", "interpolation"):
|
|
1163
|
+
continue
|
|
1164
|
+
start, end = self._spec_span(node, child, base, len(text))
|
|
1165
|
+
piece = body[cursor:max(0, start - lead)]
|
|
1166
|
+
if piece:
|
|
1167
|
+
values.append(_pos(node, ast.Constant(value=piece)))
|
|
1168
|
+
values.append(self._interpolation(child))
|
|
1169
|
+
cursor = max(cursor, end - lead)
|
|
1170
|
+
tail = body[cursor:]
|
|
1171
|
+
if tail:
|
|
1172
|
+
values.append(_pos(node, ast.Constant(value=tail)))
|
|
1173
|
+
if not values:
|
|
1174
|
+
values = [_pos(node, ast.Constant(value=body))]
|
|
1175
|
+
return _pos(node, ast.JoinedStr(values=values))
|
|
1176
|
+
|
|
1177
|
+
@staticmethod
|
|
1178
|
+
def _byte_base(node):
|
|
1179
|
+
try:
|
|
1180
|
+
return node.start_byte
|
|
1181
|
+
except Exception:
|
|
1182
|
+
return None
|
|
1183
|
+
|
|
1184
|
+
@staticmethod
|
|
1185
|
+
def _spec_span(node, child, base, size) -> Tuple[int, int]:
|
|
1186
|
+
if base is None:
|
|
1187
|
+
return 0, 0
|
|
1188
|
+
try:
|
|
1189
|
+
return max(0, min(child.start_byte - base, size)), max(0, min(child.end_byte - base, size))
|
|
1190
|
+
except Exception:
|
|
1191
|
+
return 0, 0
|
|
1192
|
+
|
|
1193
|
+
def _first_named(self, node):
|
|
1194
|
+
named = self._named(node)
|
|
1195
|
+
return named[0] if named else None
|
|
1196
|
+
|
|
1197
|
+
def _comprehension_clause(self, node) -> ast.comprehension:
|
|
1198
|
+
target = node.child_by_field_name("left")
|
|
1199
|
+
iterable = node.child_by_field_name("right")
|
|
1200
|
+
return ast.comprehension(
|
|
1201
|
+
target=self._expression(target, _STORE),
|
|
1202
|
+
iter=self._expression(iterable, _LOAD), ifs=[],
|
|
1203
|
+
is_async=1 if any(c.type == "async" for c in self._kids(node)) else 0)
|
|
1204
|
+
|
|
1205
|
+
def _comprehension(self, node, ctx, kind) -> ast.AST:
|
|
1206
|
+
body = node.child_by_field_name("body")
|
|
1207
|
+
generators: List[ast.comprehension] = []
|
|
1208
|
+
for child in self._named(node):
|
|
1209
|
+
if child.type == "for_in_clause":
|
|
1210
|
+
generators.append(self._comprehension_clause(child))
|
|
1211
|
+
elif child.type == "if_clause" and generators:
|
|
1212
|
+
condition = child.child_by_field_name("condition") or self._param_name(child)
|
|
1213
|
+
generators[-1].ifs.append(self._expression(condition, _LOAD))
|
|
1214
|
+
if not generators:
|
|
1215
|
+
# A broken comprehension still needs a traversable generator slot.
|
|
1216
|
+
generators = [ast.comprehension(target=_pos(node, ast.Name(id="_", ctx=_STORE())),
|
|
1217
|
+
iter=_pos(node, ast.Constant(None)), ifs=[], is_async=0)]
|
|
1218
|
+
if kind is ast.DictComp:
|
|
1219
|
+
key = body.child_by_field_name("key") if body is not None else None
|
|
1220
|
+
value = body.child_by_field_name("value") if body is not None else None
|
|
1221
|
+
out = ast.DictComp(key=self._expression(key, _LOAD), value=self._expression(value, _LOAD),
|
|
1222
|
+
generators=generators)
|
|
1223
|
+
else:
|
|
1224
|
+
out = kind(elt=self._expression(body, _LOAD), generators=generators)
|
|
1225
|
+
return _pos(node, out)
|
|
1226
|
+
|
|
1227
|
+
def _list_comp(self, node, ctx):
|
|
1228
|
+
return self._comprehension(node, ctx, ast.ListComp)
|
|
1229
|
+
|
|
1230
|
+
def _set_comp(self, node, ctx):
|
|
1231
|
+
return self._comprehension(node, ctx, ast.SetComp)
|
|
1232
|
+
|
|
1233
|
+
def _dict_comp(self, node, ctx):
|
|
1234
|
+
return self._comprehension(node, ctx, ast.DictComp)
|
|
1235
|
+
|
|
1236
|
+
def _gen_exp(self, node, ctx):
|
|
1237
|
+
return self._comprehension(node, ctx, ast.GeneratorExp)
|
|
1238
|
+
|
|
1239
|
+
def _lambda(self, node, ctx):
|
|
1240
|
+
return _pos(node, ast.Lambda(args=self._parameters(node.child_by_field_name("parameters")),
|
|
1241
|
+
body=self._expression(node.child_by_field_name("body"), _LOAD)))
|
|
1242
|
+
|
|
1243
|
+
def _named_expr(self, node, ctx):
|
|
1244
|
+
target = self._expression(node.child_by_field_name("name"), _STORE)
|
|
1245
|
+
if not isinstance(target, ast.Name):
|
|
1246
|
+
target = _pos(node, ast.Name(id="_", ctx=_STORE()))
|
|
1247
|
+
return _pos(node, ast.NamedExpr(target=target,
|
|
1248
|
+
value=self._expression(node.child_by_field_name("value"),
|
|
1249
|
+
_LOAD)))
|
|
1250
|
+
|
|
1251
|
+
def _yield(self, node, ctx):
|
|
1252
|
+
parts = self._named(node)
|
|
1253
|
+
value = self._expression(parts[0], _LOAD) if parts else None
|
|
1254
|
+
kind = ast.YieldFrom if any(c.type == "from" for c in self._kids(node)) else ast.Yield
|
|
1255
|
+
if kind is ast.YieldFrom and value is None:
|
|
1256
|
+
return self._placeholder(node)
|
|
1257
|
+
return _pos(node, kind(value=value))
|
|
1258
|
+
|
|
1259
|
+
def _generic_type(self, node, ctx):
|
|
1260
|
+
"""`list[T]`, `dict[str, int]`: a Subscript keeps the names reachable exactly as CPython does."""
|
|
1261
|
+
parts = self._named(node)
|
|
1262
|
+
if not parts:
|
|
1263
|
+
return self._placeholder(node)
|
|
1264
|
+
params: List[ast.AST] = []
|
|
1265
|
+
for child in parts[1:]:
|
|
1266
|
+
if child.type == "type_parameter":
|
|
1267
|
+
for param in self._named(child):
|
|
1268
|
+
made = self._unwrap_type(param)
|
|
1269
|
+
if made is not None:
|
|
1270
|
+
params.append(made)
|
|
1271
|
+
else:
|
|
1272
|
+
params.append(self._expression(child, _LOAD))
|
|
1273
|
+
value = self._expression(parts[0], _LOAD)
|
|
1274
|
+
if not params:
|
|
1275
|
+
return _pos(node, value)
|
|
1276
|
+
slice_node = params[0] if len(params) == 1 else _pos(node, ast.Tuple(
|
|
1277
|
+
elts=params, ctx=_LOAD()))
|
|
1278
|
+
return _pos(node, ast.Subscript(value=value, slice=slice_node, ctx=ctx()))
|
|
1279
|
+
|
|
1280
|
+
def _sequence(self, node, ctx, kind):
|
|
1281
|
+
elts = [self._expression(c, _LOAD) for c in self._named(node)]
|
|
1282
|
+
return _pos(node, kind(elts=elts, ctx=ctx()))
|
|
1283
|
+
|
|
1284
|
+
def _list(self, node, ctx):
|
|
1285
|
+
return self._sequence(node, ctx, ast.List)
|
|
1286
|
+
|
|
1287
|
+
def _tuple(self, node, ctx):
|
|
1288
|
+
return self._sequence(node, ctx, ast.Tuple)
|
|
1289
|
+
|
|
1290
|
+
def _set(self, node, ctx):
|
|
1291
|
+
return _pos(node, ast.Set(elts=[self._expression(c, _LOAD) for c in self._named(node)]))
|
|
1292
|
+
|
|
1293
|
+
def _tuple_of(self, node, ctx):
|
|
1294
|
+
return _pos(node, ast.Tuple(elts=[self._expression(c, ctx) for c in self._named(node)],
|
|
1295
|
+
ctx=ctx()))
|
|
1296
|
+
|
|
1297
|
+
def _list_of(self, node, ctx):
|
|
1298
|
+
return _pos(node, ast.List(elts=[self._expression(c, ctx) for c in self._named(node)],
|
|
1299
|
+
ctx=ctx()))
|
|
1300
|
+
|
|
1301
|
+
def _dict(self, node, ctx):
|
|
1302
|
+
keys: List[Optional[ast.AST]] = []
|
|
1303
|
+
values: List[ast.AST] = []
|
|
1304
|
+
for child in self._named(node):
|
|
1305
|
+
if child.type == "pair":
|
|
1306
|
+
key = child.child_by_field_name("key")
|
|
1307
|
+
value = child.child_by_field_name("value")
|
|
1308
|
+
keys.append(self._expression(key, _LOAD) if key is not None else None)
|
|
1309
|
+
values.append(self._expression(value, _LOAD) if value is not None
|
|
1310
|
+
else _pos(child, ast.Constant(None)))
|
|
1311
|
+
else:
|
|
1312
|
+
keys.append(None)
|
|
1313
|
+
values.append(self._expression(child, _LOAD))
|
|
1314
|
+
return _pos(node, ast.Dict(keys=keys, values=values))
|
|
1315
|
+
|
|
1316
|
+
def _parenthesized(self, node, ctx):
|
|
1317
|
+
inner = self._named(node)
|
|
1318
|
+
return self._expression(inner[0], ctx) if inner else self._placeholder(node)
|
|
1319
|
+
|
|
1320
|
+
def _starred(self, node, ctx):
|
|
1321
|
+
# In `a, *b = c` the star and its name are Store targets; in `f(*a)` they are Load.
|
|
1322
|
+
return _pos(node, ast.Starred(value=self._expression(self._param_name(node), ctx),
|
|
1323
|
+
ctx=ctx()))
|
|
1324
|
+
|
|
1325
|
+
def _await(self, node, ctx):
|
|
1326
|
+
inner = node.child_by_field_name("argument") or self._last_named(node)
|
|
1327
|
+
return _pos(node, ast.Await(value=self._expression(inner, _LOAD)))
|
|
1328
|
+
|
|
1329
|
+
def _conditional(self, node, ctx):
|
|
1330
|
+
# `a if c else b` -> named children [a, c, b].
|
|
1331
|
+
parts = self._named(node)
|
|
1332
|
+
if len(parts) < 3:
|
|
1333
|
+
return self._generic_expression(node, ctx)
|
|
1334
|
+
return _pos(node, ast.IfExp(test=self._expression(parts[1], _LOAD),
|
|
1335
|
+
body=self._expression(parts[0], _LOAD),
|
|
1336
|
+
orelse=self._expression(parts[2], _LOAD)))
|
|
1337
|
+
|
|
1338
|
+
_STMT = {
|
|
1339
|
+
"expression_statement": _expr_statement,
|
|
1340
|
+
"assignment": _assignment,
|
|
1341
|
+
"augmented_assignment": _assignment,
|
|
1342
|
+
"function_definition": _function,
|
|
1343
|
+
"class_definition": _class,
|
|
1344
|
+
"decorated_definition": _decorated,
|
|
1345
|
+
"return_statement": _return,
|
|
1346
|
+
"import_statement": _import,
|
|
1347
|
+
"import_from_statement": _import_from,
|
|
1348
|
+
"future_import_statement": _future_import,
|
|
1349
|
+
"if_statement": _if,
|
|
1350
|
+
"for_statement": _for,
|
|
1351
|
+
"while_statement": _while,
|
|
1352
|
+
"try_statement": _try,
|
|
1353
|
+
"with_statement": _with,
|
|
1354
|
+
"break_statement": _break,
|
|
1355
|
+
"continue_statement": _continue,
|
|
1356
|
+
"pass_statement": _pass_stmt,
|
|
1357
|
+
"raise_statement": _raise,
|
|
1358
|
+
"global_statement": _global,
|
|
1359
|
+
"nonlocal_statement": lambda self, node: self._global(node, True),
|
|
1360
|
+
"assert_statement": _assert,
|
|
1361
|
+
"delete_statement": _delete,
|
|
1362
|
+
"type_alias_statement": _type_alias,
|
|
1363
|
+
"match_statement": _match,
|
|
1364
|
+
"module": lambda self, node: self._stmt_module(node),
|
|
1365
|
+
"block": lambda self, node: self._statements(node),
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
_EXPR = {
|
|
1369
|
+
"identifier": _name,
|
|
1370
|
+
"as_pattern_target": _name,
|
|
1371
|
+
"dotted_name": _dotted,
|
|
1372
|
+
"attribute": _attribute,
|
|
1373
|
+
"subscript": _subscript,
|
|
1374
|
+
"call": _call,
|
|
1375
|
+
"string": _string,
|
|
1376
|
+
"concatenated_string": _string,
|
|
1377
|
+
"integer": _constant,
|
|
1378
|
+
"float": _constant,
|
|
1379
|
+
"true": _constant,
|
|
1380
|
+
"false": _constant,
|
|
1381
|
+
"none": _constant,
|
|
1382
|
+
"imaginary": _constant,
|
|
1383
|
+
"list": _list,
|
|
1384
|
+
"tuple": _tuple,
|
|
1385
|
+
"set": _set,
|
|
1386
|
+
"dictionary": _dict,
|
|
1387
|
+
"parenthesized_expression": _parenthesized,
|
|
1388
|
+
"binary_operator": _binary,
|
|
1389
|
+
"boolean_operator": _boolean,
|
|
1390
|
+
"comparison_operator": _comparison,
|
|
1391
|
+
"unary_operator": _unary,
|
|
1392
|
+
"not_operator": _unary,
|
|
1393
|
+
"await": _await,
|
|
1394
|
+
"conditional_expression": _conditional,
|
|
1395
|
+
"starred_expression": _starred,
|
|
1396
|
+
"list_splat_pattern": _starred,
|
|
1397
|
+
"list_splat": _starred,
|
|
1398
|
+
"lambda": _lambda,
|
|
1399
|
+
"named_expression": _named_expr,
|
|
1400
|
+
"yield": _yield,
|
|
1401
|
+
"generic_type": _generic_type,
|
|
1402
|
+
"list_comprehension": _list_comp,
|
|
1403
|
+
"set_comprehension": _set_comp,
|
|
1404
|
+
"dictionary_comprehension": _dict_comp,
|
|
1405
|
+
"generator_expression": _gen_exp,
|
|
1406
|
+
"expression_list": _tuple_of,
|
|
1407
|
+
"pattern_list": _tuple_of,
|
|
1408
|
+
"tuple_pattern": _tuple_of,
|
|
1409
|
+
"list_pattern": _list_of,
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
# PEP 634 patterns. An unmodelled pattern degrades to `MatchAs()` - a bare `_` capture, which
|
|
1413
|
+
# is always a valid pattern node, so the case block and its body stay traversable.
|
|
1414
|
+
_PATTERN = {
|
|
1415
|
+
"case_pattern": _pattern_unwrap,
|
|
1416
|
+
"group_pattern": _pattern_unwrap,
|
|
1417
|
+
"wildcard_pattern": _pattern_wildcard,
|
|
1418
|
+
"identifier": _pattern_capture,
|
|
1419
|
+
"dotted_name": _pattern_value,
|
|
1420
|
+
"attribute": _pattern_value,
|
|
1421
|
+
"string": _pattern_value,
|
|
1422
|
+
"concatenated_string": _pattern_value,
|
|
1423
|
+
"integer": _pattern_value,
|
|
1424
|
+
"float": _pattern_value,
|
|
1425
|
+
"true": _pattern_value,
|
|
1426
|
+
"false": _pattern_value,
|
|
1427
|
+
"none": _pattern_value,
|
|
1428
|
+
"list_pattern": _pattern_sequence,
|
|
1429
|
+
"tuple_pattern": _pattern_sequence,
|
|
1430
|
+
"dict_pattern": _pattern_mapping,
|
|
1431
|
+
"class_pattern": _pattern_class,
|
|
1432
|
+
"as_pattern": _pattern_as,
|
|
1433
|
+
"union_pattern": _pattern_or,
|
|
1434
|
+
"splat_pattern": _pattern_star,
|
|
1435
|
+
"list_splat_pattern": _pattern_star,
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
_BINOP = {"+": ast.Add, "-": ast.Sub, "*": ast.Mult, "/": ast.Div, "//": ast.FloorDiv,
|
|
1440
|
+
"%": ast.Mod, "**": ast.Pow, "@": ast.MatMult, "<<": ast.LShift, ">>": ast.RShift,
|
|
1441
|
+
"|": ast.BitOr, "&": ast.BitAnd, "^": ast.BitXor}
|
|
1442
|
+
_AUGOP = {k + "=": v for k, v in _BINOP.items()}
|
|
1443
|
+
_CMPOP = {"==": ast.Eq, "!=": ast.NotEq, "<": ast.Lt, "<=": ast.LtE, ">": ast.Gt, ">=": ast.GtE,
|
|
1444
|
+
"in": ast.In, "not in": ast.NotIn, "is": ast.Is, "is not": ast.IsNot}
|
|
1445
|
+
_BOOLOP = {"and": ast.And, "or": ast.Or}
|
|
1446
|
+
_UNOP = {"-": ast.USub, "+": ast.UAdd, "~": ast.Invert, "not": ast.Not}
|