cellgraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cellgraph/__init__.py +142 -0
- cellgraph/analysis.py +1013 -0
- cellgraph/core.py +87 -0
- cellgraph/diagnostics.py +26 -0
- cellgraph/effects.py +127 -0
- cellgraph/grammar.py +144 -0
- cellgraph/keys.py +420 -0
- cellgraph/languages.py +96 -0
- cellgraph/plan.py +34 -0
- cellgraph/py.typed +0 -0
- cellgraph/python.py +672 -0
- cellgraph/vocabulary.py +263 -0
- cellgraph-0.1.0.dist-info/METADATA +63 -0
- cellgraph-0.1.0.dist-info/RECORD +16 -0
- cellgraph-0.1.0.dist-info/WHEEL +4 -0
- cellgraph-0.1.0.dist-info/licenses/LICENSE +201 -0
cellgraph/python.py
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
"""The Python language analyzer: defines and references from the AST and symtable.
|
|
2
|
+
|
|
3
|
+
Ported from Strata's ``notebook/analyzer.py`` as is.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
import builtins
|
|
10
|
+
import symtable
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import TYPE_CHECKING
|
|
13
|
+
|
|
14
|
+
from cellgraph.languages import Analyzed
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from cellgraph.analysis import Cell
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class CellAnalysis:
|
|
22
|
+
"""Result of analyzing a single cell; ``error`` is None on success."""
|
|
23
|
+
|
|
24
|
+
defines: list[str] = field(default_factory=list)
|
|
25
|
+
references: list[str] = field(default_factory=list)
|
|
26
|
+
# Defines from in-place mutation (``df["col"] = ...``). The harness forces
|
|
27
|
+
# serialization of these even when the mutation preserved ``id()``.
|
|
28
|
+
mutation_defines: list[str] = field(default_factory=list)
|
|
29
|
+
# Free names that are builtins, kept out of ``references`` so every
|
|
30
|
+
# ``print``/``len`` caller doesn't list them. A cell can still define
|
|
31
|
+
# ``input = ...``, so the DAG resolves these against producers too: an
|
|
32
|
+
# unshadowed builtin wires nothing, a shadowed one gets its edge.
|
|
33
|
+
builtin_references: list[str] = field(default_factory=list)
|
|
34
|
+
error: str | None = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def imported_names(source: str) -> set[str]:
|
|
38
|
+
"""Top-level names bound by ``import`` statements in *source*.
|
|
39
|
+
|
|
40
|
+
A consumer missing one can re-import it, unlike a missing data artifact, so
|
|
41
|
+
this picks the log level for an absent upstream artifact. Binding rule
|
|
42
|
+
matches :meth:`VariableAnalyzer.visit_Import`; star imports contribute nothing.
|
|
43
|
+
"""
|
|
44
|
+
try:
|
|
45
|
+
tree = ast.parse(source)
|
|
46
|
+
except SyntaxError:
|
|
47
|
+
return set()
|
|
48
|
+
names: set[str] = set()
|
|
49
|
+
for node in tree.body:
|
|
50
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
51
|
+
for alias in node.names:
|
|
52
|
+
if alias.name == "*":
|
|
53
|
+
continue
|
|
54
|
+
names.add(alias.asname or alias.name.split(".")[0])
|
|
55
|
+
return names
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _collect_name_targets(target: ast.expr, out: set[str]) -> None:
|
|
59
|
+
"""Recursively collect Name ids from an assignment target.
|
|
60
|
+
|
|
61
|
+
Handles ``x``, tuple/list unpacking and ``*rest``. Subscript and attribute
|
|
62
|
+
targets are mutations and are ignored here.
|
|
63
|
+
"""
|
|
64
|
+
if isinstance(target, ast.Name):
|
|
65
|
+
out.add(target.id)
|
|
66
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
67
|
+
for elt in target.elts:
|
|
68
|
+
_collect_name_targets(elt, out)
|
|
69
|
+
elif isinstance(target, ast.Starred):
|
|
70
|
+
_collect_name_targets(target.value, out)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _type_param_names(params: list[ast.type_param]) -> set[str]:
|
|
74
|
+
"""Names bound by PEP 695 type parameters (``[T, *Ts, **P]``)."""
|
|
75
|
+
return {p.name for p in params if isinstance(p, (ast.TypeVar, ast.TypeVarTuple, ast.ParamSpec))}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _has_inplace_true(keywords: list[ast.keyword]) -> bool:
|
|
79
|
+
"""Return whether a call's keywords contain a literal ``inplace=True``."""
|
|
80
|
+
return any(
|
|
81
|
+
kw.arg == "inplace" and isinstance(kw.value, ast.Constant) and kw.value.value is True
|
|
82
|
+
for kw in keywords
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class VariableAnalyzer(ast.NodeVisitor):
|
|
87
|
+
"""AST visitor that collects defined and referenced variables."""
|
|
88
|
+
|
|
89
|
+
def __init__(self):
|
|
90
|
+
"""Initialize the analyzer."""
|
|
91
|
+
self.defines: set[str] = set()
|
|
92
|
+
self.references: set[str] = set()
|
|
93
|
+
# Pure ``x = ...`` targets. ``df = ...`` then ``df["col"] = ...`` in
|
|
94
|
+
# one cell is a local define, not a mutation of an upstream.
|
|
95
|
+
self.pure_defines: set[str] = set()
|
|
96
|
+
# Subscript/attribute mutations. Unlike pure rebinds these stay in
|
|
97
|
+
# references: the cell depends on an upstream producer.
|
|
98
|
+
self.mutation_defines: set[str] = set()
|
|
99
|
+
# Names read before the cell binds them (``df = df.dropna()``,
|
|
100
|
+
# ``print(x); x = 5``): genuine upstream references that survive the
|
|
101
|
+
# pure-define filter.
|
|
102
|
+
self.read_before_define: set[str] = set()
|
|
103
|
+
# Names bound so far in source order: tells ``x = 0\nx += 1``
|
|
104
|
+
# (local) from a bare ``x += 1`` (upstream read).
|
|
105
|
+
self._defined_so_far: set[str] = set()
|
|
106
|
+
self._in_nested_scope = False
|
|
107
|
+
self._local_vars: set[str] = set() # Track local scope variables
|
|
108
|
+
# PEP 563 stringifies annotations, so under it they are not
|
|
109
|
+
# runtime references.
|
|
110
|
+
self._future_annotations: bool = False
|
|
111
|
+
|
|
112
|
+
def visit_Module(self, node: ast.Module) -> None:
|
|
113
|
+
"""Visit module: process top-level statements."""
|
|
114
|
+
self._future_annotations = any(
|
|
115
|
+
isinstance(s, ast.ImportFrom)
|
|
116
|
+
and s.module == "__future__"
|
|
117
|
+
and any(alias.name == "annotations" for alias in s.names)
|
|
118
|
+
for s in node.body
|
|
119
|
+
)
|
|
120
|
+
for child in node.body:
|
|
121
|
+
self.visit(child)
|
|
122
|
+
|
|
123
|
+
def visit_Assign(self, node: ast.Assign) -> None:
|
|
124
|
+
"""Handle: x = ... or x, y = ..."""
|
|
125
|
+
all_underscore = all(self._is_pure_underscore(target) for target in node.targets)
|
|
126
|
+
if not all_underscore:
|
|
127
|
+
# The RHS runs before the targets bind, so ``x = x + 1`` and swaps
|
|
128
|
+
# like ``a, b = b, a`` read the upstream value.
|
|
129
|
+
self.visit(node.value)
|
|
130
|
+
for target in node.targets:
|
|
131
|
+
self._add_assign_target(target)
|
|
132
|
+
_collect_name_targets(target, self._defined_so_far)
|
|
133
|
+
|
|
134
|
+
def _is_pure_underscore(self, target: ast.expr) -> bool:
|
|
135
|
+
"""Check if a target is a pure _ (not part of unpacking)."""
|
|
136
|
+
if isinstance(target, ast.Name):
|
|
137
|
+
return target.id == "_"
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
def visit_AugAssign(self, node: ast.AugAssign) -> None:
|
|
141
|
+
"""Handle: x += ... or df["col"] += ...
|
|
142
|
+
|
|
143
|
+
The LHS is implicitly read, so a Name target is added to references
|
|
144
|
+
(unless the cell already pure-defined it earlier); without that the DAG
|
|
145
|
+
misses the upstream edge and the cell hits NameError. Subscript and
|
|
146
|
+
attribute targets go through ``_add_assign_target``.
|
|
147
|
+
"""
|
|
148
|
+
if isinstance(node.target, ast.Name):
|
|
149
|
+
if node.target.id not in self._defined_so_far:
|
|
150
|
+
self.references.add(node.target.id)
|
|
151
|
+
self.read_before_define.add(node.target.id)
|
|
152
|
+
self._add_assign_target(node.target)
|
|
153
|
+
self.visit(node.value)
|
|
154
|
+
if isinstance(node.target, ast.Name):
|
|
155
|
+
self._defined_so_far.add(node.target.id)
|
|
156
|
+
|
|
157
|
+
def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
|
|
158
|
+
"""Handle: x: int = ... or x: int (without value)."""
|
|
159
|
+
self._add_assign_target(node.target)
|
|
160
|
+
# Module-scope annotations are evaluated at runtime unless PEP 563.
|
|
161
|
+
if not self._future_annotations:
|
|
162
|
+
self.visit(node.annotation)
|
|
163
|
+
if node.value:
|
|
164
|
+
self.visit(node.value)
|
|
165
|
+
if isinstance(node.target, ast.Name):
|
|
166
|
+
self._defined_so_far.add(node.target.id)
|
|
167
|
+
|
|
168
|
+
def visit_Call(self, node: ast.Call) -> None:
|
|
169
|
+
"""Treat ``X.method(..., inplace=True)`` as a mutation of ``X``.
|
|
170
|
+
|
|
171
|
+
Otherwise ``df`` looks read-only, and in the shared-namespace batch path
|
|
172
|
+
a downstream cell sees the mutated object while the stored artifact holds
|
|
173
|
+
the old value. Only a literal ``inplace=True`` on an attribute call with a
|
|
174
|
+
name/attribute receiver counts.
|
|
175
|
+
"""
|
|
176
|
+
if isinstance(node.func, ast.Attribute) and _has_inplace_true(node.keywords):
|
|
177
|
+
self._add_reference_target(node.func.value)
|
|
178
|
+
self._add_mutation_define(node.func.value)
|
|
179
|
+
self.generic_visit(node)
|
|
180
|
+
|
|
181
|
+
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
|
|
182
|
+
"""Handle: def f(): the name is defined, the body is a nested scope.
|
|
183
|
+
|
|
184
|
+
Decorators, defaults and annotations (unless PEP 563 is on) evaluate at
|
|
185
|
+
module load, so their free names are module references. Body free names
|
|
186
|
+
come from the symtable pass.
|
|
187
|
+
"""
|
|
188
|
+
self.defines.add(node.name)
|
|
189
|
+
self._visit_function_signature(node)
|
|
190
|
+
self._defined_so_far.add(node.name)
|
|
191
|
+
# Don't recurse into function body (it's a nested scope)
|
|
192
|
+
|
|
193
|
+
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
|
194
|
+
"""Handle: async def f(): ..."""
|
|
195
|
+
self.defines.add(node.name)
|
|
196
|
+
self._visit_function_signature(node)
|
|
197
|
+
self._defined_so_far.add(node.name)
|
|
198
|
+
|
|
199
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
200
|
+
"""Handle: class C: the name is defined, the body is a nested scope.
|
|
201
|
+
|
|
202
|
+
Decorators, bases and class keywords evaluate at module load and are
|
|
203
|
+
walked here; body free names come from the symtable pass.
|
|
204
|
+
"""
|
|
205
|
+
self.defines.add(node.name)
|
|
206
|
+
for decorator in node.decorator_list:
|
|
207
|
+
self.visit(decorator)
|
|
208
|
+
old_local_vars = self._local_vars
|
|
209
|
+
self._local_vars = self._local_vars | _type_param_names(node.type_params)
|
|
210
|
+
for base in node.bases:
|
|
211
|
+
self.visit(base)
|
|
212
|
+
for kw in node.keywords:
|
|
213
|
+
self.visit(kw.value)
|
|
214
|
+
self._local_vars = old_local_vars
|
|
215
|
+
self._defined_so_far.add(node.name)
|
|
216
|
+
# Don't recurse into class body
|
|
217
|
+
|
|
218
|
+
def _visit_function_signature(
|
|
219
|
+
self,
|
|
220
|
+
node: ast.FunctionDef | ast.AsyncFunctionDef,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Visit decorators, defaults and (without PEP 563) annotations of a function."""
|
|
223
|
+
for decorator in node.decorator_list:
|
|
224
|
+
self.visit(decorator)
|
|
225
|
+
for default in node.args.defaults:
|
|
226
|
+
self.visit(default)
|
|
227
|
+
for default in node.args.kw_defaults:
|
|
228
|
+
if default is not None:
|
|
229
|
+
self.visit(default)
|
|
230
|
+
if not self._future_annotations:
|
|
231
|
+
# PEP 695 type parameters are local to the signature's annotations.
|
|
232
|
+
old_local_vars = self._local_vars
|
|
233
|
+
self._local_vars = self._local_vars | _type_param_names(node.type_params)
|
|
234
|
+
if node.returns is not None:
|
|
235
|
+
self.visit(node.returns)
|
|
236
|
+
for arg_list in (node.args.posonlyargs, node.args.args, node.args.kwonlyargs):
|
|
237
|
+
for arg in arg_list:
|
|
238
|
+
if arg.annotation is not None:
|
|
239
|
+
self.visit(arg.annotation)
|
|
240
|
+
if node.args.vararg is not None and node.args.vararg.annotation is not None:
|
|
241
|
+
self.visit(node.args.vararg.annotation)
|
|
242
|
+
if node.args.kwarg is not None and node.args.kwarg.annotation is not None:
|
|
243
|
+
self.visit(node.args.kwarg.annotation)
|
|
244
|
+
self._local_vars = old_local_vars
|
|
245
|
+
|
|
246
|
+
def visit_Import(self, node: ast.Import) -> None:
|
|
247
|
+
"""Handle: import foo, import foo as bar, or import foo.bar (binds ``foo``)."""
|
|
248
|
+
for alias in node.names:
|
|
249
|
+
name = alias.asname if alias.asname else alias.name.split(".")[0]
|
|
250
|
+
self.defines.add(name)
|
|
251
|
+
self._defined_so_far.add(name)
|
|
252
|
+
|
|
253
|
+
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
254
|
+
"""Handle: from foo import bar or from foo import *."""
|
|
255
|
+
for alias in node.names:
|
|
256
|
+
if alias.name == "*":
|
|
257
|
+
# Star imports: defines are unknowable, skip.
|
|
258
|
+
pass
|
|
259
|
+
else:
|
|
260
|
+
name = alias.asname if alias.asname else alias.name
|
|
261
|
+
self.defines.add(name)
|
|
262
|
+
self._defined_so_far.add(name)
|
|
263
|
+
|
|
264
|
+
def visit_For(self, node: ast.For) -> None:
|
|
265
|
+
"""Handle ``for x in ...``: x is defined at top level."""
|
|
266
|
+
self._add_assign_target(node.target)
|
|
267
|
+
self.visit(node.iter)
|
|
268
|
+
_collect_name_targets(node.target, self._defined_so_far)
|
|
269
|
+
for stmt in node.body:
|
|
270
|
+
self.visit(stmt)
|
|
271
|
+
for stmt in node.orelse:
|
|
272
|
+
self.visit(stmt)
|
|
273
|
+
|
|
274
|
+
def visit_With(self, node: ast.With) -> None:
|
|
275
|
+
"""Handle ``with ... as x``: x is defined at top level."""
|
|
276
|
+
for item in node.items:
|
|
277
|
+
if item.optional_vars:
|
|
278
|
+
self._add_assign_target(item.optional_vars)
|
|
279
|
+
self.visit(item.context_expr)
|
|
280
|
+
if item.optional_vars:
|
|
281
|
+
_collect_name_targets(item.optional_vars, self._defined_so_far)
|
|
282
|
+
for stmt in node.body:
|
|
283
|
+
self.visit(stmt)
|
|
284
|
+
|
|
285
|
+
def visit_AsyncWith(self, node: ast.AsyncWith) -> None:
|
|
286
|
+
"""Handle: async with ... as x:"""
|
|
287
|
+
for item in node.items:
|
|
288
|
+
if item.optional_vars:
|
|
289
|
+
self._add_assign_target(item.optional_vars)
|
|
290
|
+
self.visit(item.context_expr)
|
|
291
|
+
if item.optional_vars:
|
|
292
|
+
_collect_name_targets(item.optional_vars, self._defined_so_far)
|
|
293
|
+
for stmt in node.body:
|
|
294
|
+
self.visit(stmt)
|
|
295
|
+
|
|
296
|
+
def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None:
|
|
297
|
+
"""Handle ``except E as e``: e is defined at top level."""
|
|
298
|
+
if node.name:
|
|
299
|
+
self.defines.add(node.name)
|
|
300
|
+
if node.type:
|
|
301
|
+
self.visit(node.type)
|
|
302
|
+
if node.name:
|
|
303
|
+
self._defined_so_far.add(node.name)
|
|
304
|
+
for stmt in node.body:
|
|
305
|
+
self.visit(stmt)
|
|
306
|
+
|
|
307
|
+
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
|
|
308
|
+
"""Handle: walrus operator := at top level."""
|
|
309
|
+
if isinstance(node.target, ast.Name):
|
|
310
|
+
self.defines.add(node.target.id)
|
|
311
|
+
self.visit(node.value)
|
|
312
|
+
if isinstance(node.target, ast.Name):
|
|
313
|
+
self._defined_so_far.add(node.target.id)
|
|
314
|
+
|
|
315
|
+
def visit_TypeAlias(self, node: ast.TypeAlias) -> None:
|
|
316
|
+
"""Handle: type A = ... (PEP 695); the lazy value's names come from symtable."""
|
|
317
|
+
self._add_assign_target(node.name)
|
|
318
|
+
_collect_name_targets(node.name, self._defined_so_far)
|
|
319
|
+
|
|
320
|
+
def visit_MatchAs(self, node: ast.MatchAs) -> None:
|
|
321
|
+
"""Handle: case [x] / case P() as x: the capture binds x."""
|
|
322
|
+
if node.pattern is not None:
|
|
323
|
+
self.visit(node.pattern)
|
|
324
|
+
if node.name is not None:
|
|
325
|
+
self._bind_capture(node.name)
|
|
326
|
+
|
|
327
|
+
def visit_MatchStar(self, node: ast.MatchStar) -> None:
|
|
328
|
+
"""Handle: case [first, *rest]."""
|
|
329
|
+
if node.name is not None:
|
|
330
|
+
self._bind_capture(node.name)
|
|
331
|
+
|
|
332
|
+
def visit_MatchMapping(self, node: ast.MatchMapping) -> None:
|
|
333
|
+
"""Handle: case {"k": v, **rest}."""
|
|
334
|
+
for key in node.keys:
|
|
335
|
+
self.visit(key)
|
|
336
|
+
for pattern in node.patterns:
|
|
337
|
+
self.visit(pattern)
|
|
338
|
+
if node.rest is not None:
|
|
339
|
+
self._bind_capture(node.rest)
|
|
340
|
+
|
|
341
|
+
def _bind_capture(self, name: str) -> None:
|
|
342
|
+
self.defines.add(name)
|
|
343
|
+
self.pure_defines.add(name)
|
|
344
|
+
self._defined_so_far.add(name)
|
|
345
|
+
|
|
346
|
+
def visit_Delete(self, node: ast.Delete) -> None:
|
|
347
|
+
"""Handle ``del x``: x is referenced but not defined."""
|
|
348
|
+
for target in node.targets:
|
|
349
|
+
self._add_delete_target(target)
|
|
350
|
+
|
|
351
|
+
def _add_delete_target(self, target: ast.expr) -> None:
|
|
352
|
+
"""Add the root name of a ``del`` target (``x``, ``x.attr``, ``x[k]``) to references."""
|
|
353
|
+
if isinstance(target, ast.Name):
|
|
354
|
+
self.references.add(target.id)
|
|
355
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
356
|
+
for elt in target.elts:
|
|
357
|
+
self._add_delete_target(elt)
|
|
358
|
+
elif isinstance(target, ast.Subscript):
|
|
359
|
+
self._add_delete_target(target.value)
|
|
360
|
+
self.visit(target.slice)
|
|
361
|
+
elif isinstance(target, ast.Attribute):
|
|
362
|
+
self._add_delete_target(target.value)
|
|
363
|
+
|
|
364
|
+
def visit_Name(self, node: ast.Name) -> None:
|
|
365
|
+
"""Collect referenced names (Load context)."""
|
|
366
|
+
if isinstance(node.ctx, ast.Load):
|
|
367
|
+
# Skip names local to the current scope (e.g. lambda params)
|
|
368
|
+
if node.id not in self._local_vars:
|
|
369
|
+
self.references.add(node.id)
|
|
370
|
+
if node.id not in self._defined_so_far:
|
|
371
|
+
self.read_before_define.add(node.id)
|
|
372
|
+
|
|
373
|
+
def visit_ListComp(self, node: ast.ListComp) -> None:
|
|
374
|
+
"""``[elt for x in iter if cond]``: loop targets are local; the first iter is outer."""
|
|
375
|
+
self._visit_comprehension(node, [node.elt])
|
|
376
|
+
|
|
377
|
+
def visit_SetComp(self, node: ast.SetComp) -> None:
|
|
378
|
+
"""``{elt for x in iter}``."""
|
|
379
|
+
self._visit_comprehension(node, [node.elt])
|
|
380
|
+
|
|
381
|
+
def visit_DictComp(self, node: ast.DictComp) -> None:
|
|
382
|
+
"""``{k: v for x in iter}``: both key and value are scoped."""
|
|
383
|
+
self._visit_comprehension(node, [node.key, node.value])
|
|
384
|
+
|
|
385
|
+
def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
|
|
386
|
+
"""``(elt for x in iter)``."""
|
|
387
|
+
self._visit_comprehension(node, [node.elt])
|
|
388
|
+
|
|
389
|
+
def _visit_comprehension(
|
|
390
|
+
self,
|
|
391
|
+
node: ast.ListComp | ast.SetComp | ast.DictComp | ast.GeneratorExp,
|
|
392
|
+
elements: list[ast.expr],
|
|
393
|
+
) -> None:
|
|
394
|
+
"""Walk comprehension parts with loop variables locally scoped.
|
|
395
|
+
|
|
396
|
+
The first iterable runs in the outer scope; later iterables, elements
|
|
397
|
+
and ``if`` clauses see the loop variables. Python 3.13 inlines
|
|
398
|
+
comprehensions (PEP 709), so symtable has no child scope for them and
|
|
399
|
+
this AST tracking is what finds their free names.
|
|
400
|
+
"""
|
|
401
|
+
if not node.generators:
|
|
402
|
+
return
|
|
403
|
+
|
|
404
|
+
# First generator's iter runs in the OUTER scope.
|
|
405
|
+
self.visit(node.generators[0].iter)
|
|
406
|
+
|
|
407
|
+
old_local_vars = self._local_vars
|
|
408
|
+
try:
|
|
409
|
+
local_vars = set(self._local_vars)
|
|
410
|
+
for gen in node.generators:
|
|
411
|
+
local_vars |= self._extract_target_names(gen.target)
|
|
412
|
+
self._local_vars = local_vars
|
|
413
|
+
|
|
414
|
+
# Element(s) and if-clauses see all loop targets.
|
|
415
|
+
for elt in elements:
|
|
416
|
+
self.visit(elt)
|
|
417
|
+
# Subsequent generators' iter expressions can see prior
|
|
418
|
+
# loop targets, so they get visited under the local scope.
|
|
419
|
+
for gen in node.generators[1:]:
|
|
420
|
+
self.visit(gen.iter)
|
|
421
|
+
for gen in node.generators:
|
|
422
|
+
for cond in gen.ifs:
|
|
423
|
+
self.visit(cond)
|
|
424
|
+
finally:
|
|
425
|
+
self._local_vars = old_local_vars
|
|
426
|
+
|
|
427
|
+
def _extract_target_names(self, target: ast.expr) -> set[str]:
|
|
428
|
+
"""Names bound by a comprehension or for-loop target."""
|
|
429
|
+
names: set[str] = set()
|
|
430
|
+
if isinstance(target, ast.Name):
|
|
431
|
+
names.add(target.id)
|
|
432
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
433
|
+
for elt in target.elts:
|
|
434
|
+
names |= self._extract_target_names(elt)
|
|
435
|
+
elif isinstance(target, ast.Starred):
|
|
436
|
+
names |= self._extract_target_names(target.value)
|
|
437
|
+
return names
|
|
438
|
+
|
|
439
|
+
def visit_Lambda(self, node: ast.Lambda) -> None:
|
|
440
|
+
"""Lambda expression: arguments are local scope."""
|
|
441
|
+
# Defaults evaluate in the enclosing scope, before params shadow.
|
|
442
|
+
for default in node.args.defaults:
|
|
443
|
+
self.visit(default)
|
|
444
|
+
for default in node.args.kw_defaults:
|
|
445
|
+
if default is not None:
|
|
446
|
+
self.visit(default)
|
|
447
|
+
old_local_vars = self._local_vars
|
|
448
|
+
self._local_vars = self._local_vars | self._get_lambda_params(node.args)
|
|
449
|
+
self.visit(node.body)
|
|
450
|
+
self._local_vars = old_local_vars
|
|
451
|
+
|
|
452
|
+
def _get_lambda_params(self, args: ast.arguments) -> set[str]:
|
|
453
|
+
"""Extract parameter names from lambda arguments."""
|
|
454
|
+
params = set()
|
|
455
|
+
for arg in args.posonlyargs:
|
|
456
|
+
params.add(arg.arg)
|
|
457
|
+
for arg in args.args:
|
|
458
|
+
params.add(arg.arg)
|
|
459
|
+
if args.vararg:
|
|
460
|
+
params.add(args.vararg.arg)
|
|
461
|
+
for arg in args.kwonlyargs:
|
|
462
|
+
params.add(arg.arg)
|
|
463
|
+
if args.kwarg:
|
|
464
|
+
params.add(args.kwarg.arg)
|
|
465
|
+
return params
|
|
466
|
+
|
|
467
|
+
def _add_assign_target(self, target: ast.expr) -> None:
|
|
468
|
+
"""Add the names an assignment target defines.
|
|
469
|
+
|
|
470
|
+
Subscript (``df["col"]``) and attribute (``obj.attr``) targets define
|
|
471
|
+
their root name.
|
|
472
|
+
"""
|
|
473
|
+
if isinstance(target, ast.Name):
|
|
474
|
+
self.defines.add(target.id)
|
|
475
|
+
self.pure_defines.add(target.id)
|
|
476
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
477
|
+
for elt in target.elts:
|
|
478
|
+
self._add_assign_target(elt)
|
|
479
|
+
elif isinstance(target, ast.Subscript):
|
|
480
|
+
# df["col"] = ...: reads df and produces the mutated df, so
|
|
481
|
+
# downstream reads route through this cell (else they can run
|
|
482
|
+
# before the mutation and KeyError).
|
|
483
|
+
self._add_reference_target(target.value)
|
|
484
|
+
self._add_mutation_define(target.value)
|
|
485
|
+
elif isinstance(target, ast.Attribute):
|
|
486
|
+
# obj.attr = ...: same as subscript mutation.
|
|
487
|
+
self._add_reference_target(target.value)
|
|
488
|
+
self._add_mutation_define(target.value)
|
|
489
|
+
elif isinstance(target, ast.Starred):
|
|
490
|
+
self._add_assign_target(target.value)
|
|
491
|
+
|
|
492
|
+
def _add_reference_target(self, node: ast.expr) -> None:
|
|
493
|
+
"""Add the root name of an attribute/subscript mutation to references."""
|
|
494
|
+
if isinstance(node, ast.Name):
|
|
495
|
+
self.references.add(node.id)
|
|
496
|
+
elif isinstance(node, (ast.Attribute, ast.Subscript)):
|
|
497
|
+
self._add_reference_target(node.value)
|
|
498
|
+
|
|
499
|
+
def _add_mutation_define(self, node: ast.expr) -> None:
|
|
500
|
+
"""Record the root name of a mutated target as a define.
|
|
501
|
+
|
|
502
|
+
The mutating cell produces the post-mutation value downstream cells see.
|
|
503
|
+
The name also stays in references (the mutation reads the prior value);
|
|
504
|
+
``mutation_defines`` tells the caller not to strip it from them.
|
|
505
|
+
"""
|
|
506
|
+
if isinstance(node, ast.Name):
|
|
507
|
+
self.defines.add(node.id)
|
|
508
|
+
self.mutation_defines.add(node.id)
|
|
509
|
+
elif isinstance(node, (ast.Attribute, ast.Subscript)):
|
|
510
|
+
self._add_mutation_define(node.value)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _collect_body_refs(source: str) -> set[str]:
|
|
514
|
+
"""Names referenced inside function/class bodies that resolve via module globals.
|
|
515
|
+
|
|
516
|
+
The AST visitor stops at body boundaries; ``symtable`` does the real scope
|
|
517
|
+
analysis. The caller filters builtins, privates and defines.
|
|
518
|
+
"""
|
|
519
|
+
try:
|
|
520
|
+
root = symtable.symtable(source, "<cell>", "exec")
|
|
521
|
+
except SyntaxError:
|
|
522
|
+
return set()
|
|
523
|
+
|
|
524
|
+
refs: set[str] = set()
|
|
525
|
+
for child in root.get_children():
|
|
526
|
+
_walk_body(child, refs)
|
|
527
|
+
return refs
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _walk_body(scope: symtable.SymbolTable, refs: set[str]) -> None:
|
|
531
|
+
"""Recursively collect names that fall through to module globals.
|
|
532
|
+
|
|
533
|
+
A symbol does iff it is referenced, ``is_global()``, and not local, a
|
|
534
|
+
parameter or a closure variable (``is_free()`` resolves via enclosing scopes).
|
|
535
|
+
"""
|
|
536
|
+
for sym in scope.get_symbols():
|
|
537
|
+
if (
|
|
538
|
+
sym.is_referenced()
|
|
539
|
+
and sym.is_global()
|
|
540
|
+
and not sym.is_local()
|
|
541
|
+
and not sym.is_parameter()
|
|
542
|
+
and not sym.is_free()
|
|
543
|
+
):
|
|
544
|
+
refs.add(sym.get_name())
|
|
545
|
+
for inner in scope.get_children():
|
|
546
|
+
_walk_body(inner, refs)
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _collect_global_writes(source: str) -> tuple[set[str], set[str]]:
|
|
550
|
+
"""Names a function binds at module scope through ``global X; X = ...``.
|
|
551
|
+
|
|
552
|
+
The AST visitor only sees module-level defines, so it misses these.
|
|
553
|
+
|
|
554
|
+
Returns ``(writes, writes_with_reads)``; the second subset is also read in
|
|
555
|
+
the function and stays in references, so the DAG records the cell as both
|
|
556
|
+
consumer and producer. A bare ``global X`` without an assign, and
|
|
557
|
+
``nonlocal``, bind nothing at module scope and are skipped.
|
|
558
|
+
"""
|
|
559
|
+
try:
|
|
560
|
+
root = symtable.symtable(source, "<cell>", "exec")
|
|
561
|
+
except SyntaxError:
|
|
562
|
+
return set(), set()
|
|
563
|
+
writes: set[str] = set()
|
|
564
|
+
read_writes: set[str] = set()
|
|
565
|
+
for child in root.get_children():
|
|
566
|
+
_walk_global_writes(child, writes, read_writes)
|
|
567
|
+
return writes, read_writes
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _walk_global_writes(
|
|
571
|
+
scope: symtable.SymbolTable,
|
|
572
|
+
writes: set[str],
|
|
573
|
+
read_writes: set[str],
|
|
574
|
+
) -> None:
|
|
575
|
+
for sym in scope.get_symbols():
|
|
576
|
+
if sym.is_assigned() and sym.is_declared_global():
|
|
577
|
+
writes.add(sym.get_name())
|
|
578
|
+
if sym.is_referenced():
|
|
579
|
+
read_writes.add(sym.get_name())
|
|
580
|
+
for inner in scope.get_children():
|
|
581
|
+
_walk_global_writes(inner, writes, read_writes)
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def analyze_cell(source: str) -> CellAnalysis:
|
|
585
|
+
"""Analyze a cell's source and extract its defines and references."""
|
|
586
|
+
if not source or not source.strip():
|
|
587
|
+
return CellAnalysis(defines=[], references=[])
|
|
588
|
+
|
|
589
|
+
try:
|
|
590
|
+
tree = ast.parse(source)
|
|
591
|
+
except SyntaxError as e:
|
|
592
|
+
return CellAnalysis(
|
|
593
|
+
defines=[],
|
|
594
|
+
references=[],
|
|
595
|
+
error=f"Syntax error: {e.msg}",
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
# The visitor stops at function/class bodies; symtable covers those.
|
|
599
|
+
analyzer = VariableAnalyzer()
|
|
600
|
+
analyzer.visit(tree)
|
|
601
|
+
|
|
602
|
+
nested_refs = _collect_body_refs(source)
|
|
603
|
+
|
|
604
|
+
# ``global`` writes inside functions are module defines too.
|
|
605
|
+
# ``global_read_writes`` are also read there, so they stay in references.
|
|
606
|
+
global_writes, global_read_writes = _collect_global_writes(source)
|
|
607
|
+
|
|
608
|
+
builtin_names = set(dir(builtins)) | {"__name__", "__file__", "__doc__", "__package__"}
|
|
609
|
+
defines = [v for v in (analyzer.defines | global_writes) if not v.startswith("_")]
|
|
610
|
+
# A pure assignment in the same cell supersedes a mutation-define.
|
|
611
|
+
effective_mutation_defines = {
|
|
612
|
+
v
|
|
613
|
+
for v in (analyzer.mutation_defines | global_read_writes)
|
|
614
|
+
if not v.startswith("_") and v in set(defines) and v not in analyzer.pure_defines
|
|
615
|
+
}
|
|
616
|
+
# Pure defines are filtered from references (intra-cell rebinds);
|
|
617
|
+
# mutation-defines and reads before the cell's own binding are kept.
|
|
618
|
+
pure_defined_names = set(defines) - effective_mutation_defines - analyzer.read_before_define
|
|
619
|
+
combined_refs = set(analyzer.references) | nested_refs
|
|
620
|
+
references = [
|
|
621
|
+
v
|
|
622
|
+
for v in combined_refs
|
|
623
|
+
if not v.startswith("_") and v not in builtin_names and v not in pure_defined_names
|
|
624
|
+
]
|
|
625
|
+
builtin_references = [
|
|
626
|
+
v
|
|
627
|
+
for v in combined_refs
|
|
628
|
+
if not v.startswith("_") and v in builtin_names and v not in pure_defined_names
|
|
629
|
+
]
|
|
630
|
+
|
|
631
|
+
defines = sorted(set(defines))
|
|
632
|
+
references = sorted(set(references))
|
|
633
|
+
mutation_defines = sorted(effective_mutation_defines)
|
|
634
|
+
|
|
635
|
+
return CellAnalysis(
|
|
636
|
+
defines=defines,
|
|
637
|
+
references=references,
|
|
638
|
+
mutation_defines=mutation_defines,
|
|
639
|
+
builtin_references=sorted(set(builtin_references)),
|
|
640
|
+
error=None,
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def normalize(source: str) -> str:
|
|
645
|
+
"""Canonical text for *source*: its AST round trip, so comments and layout drop out.
|
|
646
|
+
|
|
647
|
+
Source that does not parse falls back to the line strip, which keeps comments.
|
|
648
|
+
"""
|
|
649
|
+
try:
|
|
650
|
+
return ast.unparse(ast.parse(source))
|
|
651
|
+
except SyntaxError:
|
|
652
|
+
lines = [line.rstrip() for line in source.splitlines()]
|
|
653
|
+
return "\n".join(lines).strip()
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
class PythonAnalyzer:
|
|
657
|
+
"""The Python language analyzer: ``analyze_cell`` over the cell body."""
|
|
658
|
+
|
|
659
|
+
owns_key = True
|
|
660
|
+
|
|
661
|
+
def analyze(self, cell: Cell) -> Analyzed:
|
|
662
|
+
result = analyze_cell(cell.source)
|
|
663
|
+
return Analyzed(
|
|
664
|
+
defines=list(result.defines),
|
|
665
|
+
references=list(result.references),
|
|
666
|
+
builtin_references=list(result.builtin_references),
|
|
667
|
+
mutation_defines=list(result.mutation_defines),
|
|
668
|
+
error=result.error,
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
def normalize(self, body: str) -> str:
|
|
672
|
+
return normalize(body)
|