coltrace 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coltrace/__init__.py +9 -0
- coltrace/__main__.py +6 -0
- coltrace/analysis.py +2206 -0
- coltrace/callbacks.py +579 -0
- coltrace/cli.py +668 -0
- coltrace/collector.py +561 -0
- coltrace/exporters.py +676 -0
- coltrace/graph.py +380 -0
- coltrace/history.py +441 -0
- coltrace/journey.py +578 -0
- coltrace/lineage_models.py +112 -0
- coltrace/models.py +92 -0
- coltrace/py.typed +0 -0
- coltrace/scanner.py +236 -0
- coltrace/templates/graph.html +418 -0
- coltrace/visual_layout.py +159 -0
- coltrace-0.1.0.dev4.dist-info/METADATA +241 -0
- coltrace-0.1.0.dev4.dist-info/RECORD +22 -0
- coltrace-0.1.0.dev4.dist-info/WHEEL +5 -0
- coltrace-0.1.0.dev4.dist-info/entry_points.txt +2 -0
- coltrace-0.1.0.dev4.dist-info/licenses/LICENSE +21 -0
- coltrace-0.1.0.dev4.dist-info/top_level.txt +1 -0
coltrace/analysis.py
ADDED
|
@@ -0,0 +1,2206 @@
|
|
|
1
|
+
"""Conservative, ordered interpretation of a deliberately small Pandas subset.
|
|
2
|
+
|
|
3
|
+
Only ASTs and abstract values are interpreted. Project code and data sources are
|
|
4
|
+
never imported, evaluated, or opened by this module.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ast
|
|
10
|
+
import hashlib
|
|
11
|
+
import io
|
|
12
|
+
import tokenize
|
|
13
|
+
from dataclasses import dataclass, field, replace
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Literal
|
|
16
|
+
|
|
17
|
+
from .callbacks import summarize_callback
|
|
18
|
+
from .lineage_models import (
|
|
19
|
+
Column,
|
|
20
|
+
Completeness,
|
|
21
|
+
Confidence,
|
|
22
|
+
Consumption,
|
|
23
|
+
Edge,
|
|
24
|
+
Frame,
|
|
25
|
+
HistoryEvent,
|
|
26
|
+
LineageReport,
|
|
27
|
+
Operation,
|
|
28
|
+
)
|
|
29
|
+
from .models import Diagnostic, Location
|
|
30
|
+
from .scanner import ScanConfig, scan_project
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _identifier(prefix: str, *parts: object) -> str:
|
|
34
|
+
digest = hashlib.sha256(repr(parts).encode("utf-8")).hexdigest()[:24]
|
|
35
|
+
return f"{prefix}:{digest}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _confidence(*values: Confidence) -> Confidence:
|
|
39
|
+
if "unresolved" in values:
|
|
40
|
+
return "unresolved"
|
|
41
|
+
return "inferred" if "inferred" in values else "confirmed"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _completeness(*values: Completeness) -> Completeness:
|
|
45
|
+
if "unresolved" in values:
|
|
46
|
+
return "partial" if "complete" in values or "partial" in values else "unresolved"
|
|
47
|
+
return "partial" if "partial" in values else "complete"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class _Value:
|
|
52
|
+
kind: Literal[
|
|
53
|
+
"unknown",
|
|
54
|
+
"literal",
|
|
55
|
+
"series",
|
|
56
|
+
"frame",
|
|
57
|
+
"symbol",
|
|
58
|
+
"function",
|
|
59
|
+
"rows",
|
|
60
|
+
"index",
|
|
61
|
+
"mask",
|
|
62
|
+
"accessor",
|
|
63
|
+
] = "unknown"
|
|
64
|
+
frame: _State | None = None
|
|
65
|
+
dependencies: tuple[str, ...] = ()
|
|
66
|
+
symbol: str = ""
|
|
67
|
+
function: _Function | None = None
|
|
68
|
+
evidence: Confidence = "confirmed"
|
|
69
|
+
completeness: Completeness = "unresolved"
|
|
70
|
+
escaped: tuple[str, ...] = ()
|
|
71
|
+
epochs: tuple[tuple[str, int], ...] = ()
|
|
72
|
+
components: tuple[_Value, ...] | None = None
|
|
73
|
+
index_key: str | None = None
|
|
74
|
+
producer: str = "derive"
|
|
75
|
+
expression: str = ""
|
|
76
|
+
history: tuple[str, ...] = ()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class _Module:
|
|
81
|
+
name: str
|
|
82
|
+
path: str
|
|
83
|
+
source: str
|
|
84
|
+
tree: ast.Module
|
|
85
|
+
env: dict[str, _Value] = field(default_factory=dict)
|
|
86
|
+
status: Literal["new", "visiting", "done"] = "new"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class _Function:
|
|
91
|
+
module: _Module
|
|
92
|
+
node: ast.FunctionDef
|
|
93
|
+
defaults: dict[str, _Value]
|
|
94
|
+
frame_parameters: set[str]
|
|
95
|
+
called: bool = False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class _Context:
|
|
100
|
+
module: _Module
|
|
101
|
+
scope: str
|
|
102
|
+
key: str
|
|
103
|
+
env: dict[str, _Value]
|
|
104
|
+
stack: tuple[tuple[str, str], ...] = ()
|
|
105
|
+
parent_history: tuple[str, ...] = ()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass
|
|
109
|
+
class _Snapshot:
|
|
110
|
+
state: _State
|
|
111
|
+
current: dict[str, str]
|
|
112
|
+
fallback: _Copy | None
|
|
113
|
+
schema: set[str] | None
|
|
114
|
+
epoch: int
|
|
115
|
+
tainted: bool
|
|
116
|
+
versions: dict[str, int]
|
|
117
|
+
removed: set[str]
|
|
118
|
+
history: tuple[str, ...]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass
|
|
122
|
+
class _Copy:
|
|
123
|
+
source: _Snapshot
|
|
124
|
+
names: dict[str, str]
|
|
125
|
+
context: _Context
|
|
126
|
+
node: ast.AST
|
|
127
|
+
evidence: Confidence
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass
|
|
131
|
+
class _State:
|
|
132
|
+
record: Frame
|
|
133
|
+
current: dict[str, str] = field(default_factory=dict)
|
|
134
|
+
versions: dict[str, int] = field(default_factory=dict)
|
|
135
|
+
fallback: _Copy | None = None
|
|
136
|
+
schema: set[str] | None = None
|
|
137
|
+
epoch: int = 0
|
|
138
|
+
tainted: bool = False
|
|
139
|
+
removed: set[str] = field(default_factory=set)
|
|
140
|
+
index_key: str = ""
|
|
141
|
+
positional: tuple[_Value, ...] | None = None
|
|
142
|
+
history: tuple[str, ...] = ()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _Analyzer:
|
|
146
|
+
MAX_CALLS = 1000
|
|
147
|
+
MAX_DEPTH = 32
|
|
148
|
+
|
|
149
|
+
def __init__(self, modules: dict[str, _Module], ambiguous: set[str]) -> None:
|
|
150
|
+
self.modules = modules
|
|
151
|
+
self.ambiguous = ambiguous
|
|
152
|
+
self.frames: list[Frame] = []
|
|
153
|
+
self.states: dict[str, _State] = {}
|
|
154
|
+
self.columns: dict[str, Column] = {}
|
|
155
|
+
self.operations: list[Operation] = []
|
|
156
|
+
self.edges: list[Edge] = []
|
|
157
|
+
self.consumptions: list[Consumption] = []
|
|
158
|
+
self.diagnostics: list[Diagnostic] = []
|
|
159
|
+
self.functions: list[_Function] = []
|
|
160
|
+
self.call_count = 0
|
|
161
|
+
self.api_mutations: set[str] = set()
|
|
162
|
+
self.history: list[HistoryEvent] = []
|
|
163
|
+
self.column_history: dict[str, str] = {}
|
|
164
|
+
self.frame_boundaries: dict[str, str] = {}
|
|
165
|
+
|
|
166
|
+
@staticmethod
|
|
167
|
+
def _code(context: _Context, node: ast.AST) -> str:
|
|
168
|
+
return ast.get_source_segment(context.module.source, node) or ast.unparse(node)
|
|
169
|
+
|
|
170
|
+
def _event(
|
|
171
|
+
self,
|
|
172
|
+
context: _Context,
|
|
173
|
+
node: ast.AST,
|
|
174
|
+
kind: str,
|
|
175
|
+
label: str,
|
|
176
|
+
*,
|
|
177
|
+
frames: tuple[str, ...] = (),
|
|
178
|
+
inputs: tuple[str, ...] = (),
|
|
179
|
+
outputs: tuple[str, ...] = (),
|
|
180
|
+
parents: tuple[str, ...] = (),
|
|
181
|
+
expression: str | None = None,
|
|
182
|
+
evidence: Confidence = "confirmed",
|
|
183
|
+
completeness: Completeness = "complete",
|
|
184
|
+
) -> str:
|
|
185
|
+
sequence = len(self.history)
|
|
186
|
+
evidence = _confidence(
|
|
187
|
+
evidence, "inferred" if context.key.startswith("template:") else "confirmed"
|
|
188
|
+
)
|
|
189
|
+
parents = tuple(sorted(set(parents)))
|
|
190
|
+
event = HistoryEvent(
|
|
191
|
+
_identifier("history", context.key, self._location(context, node), kind, sequence),
|
|
192
|
+
sequence,
|
|
193
|
+
kind,
|
|
194
|
+
label,
|
|
195
|
+
self._location(context, node),
|
|
196
|
+
context.module.name,
|
|
197
|
+
context.scope,
|
|
198
|
+
context.key,
|
|
199
|
+
tuple(sorted(set(frames))),
|
|
200
|
+
tuple(sorted(set(inputs))),
|
|
201
|
+
tuple(sorted(set(outputs))),
|
|
202
|
+
self._code(context, node) if expression is None else expression,
|
|
203
|
+
evidence,
|
|
204
|
+
completeness,
|
|
205
|
+
parents,
|
|
206
|
+
)
|
|
207
|
+
self.history.append(event)
|
|
208
|
+
return event.id
|
|
209
|
+
|
|
210
|
+
def _parents(self, value: _Value) -> tuple[str, ...]:
|
|
211
|
+
return tuple(
|
|
212
|
+
dict.fromkeys(
|
|
213
|
+
value.history
|
|
214
|
+
+ tuple(
|
|
215
|
+
self.column_history[item]
|
|
216
|
+
for item in value.dependencies
|
|
217
|
+
if item in self.column_history
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
@staticmethod
|
|
223
|
+
def _location(context: _Context, node: ast.AST) -> Location:
|
|
224
|
+
lines = context.module.source.split("\n")
|
|
225
|
+
|
|
226
|
+
def offset(line: int, byte: int) -> int:
|
|
227
|
+
if not 1 <= line <= len(lines):
|
|
228
|
+
return byte
|
|
229
|
+
return len(lines[line - 1].encode("utf-8")[:byte].decode("utf-8", errors="ignore"))
|
|
230
|
+
|
|
231
|
+
line = getattr(node, "lineno", 1)
|
|
232
|
+
end_line = getattr(node, "end_lineno", None)
|
|
233
|
+
end_col = getattr(node, "end_col_offset", None)
|
|
234
|
+
return Location(
|
|
235
|
+
context.module.path,
|
|
236
|
+
line,
|
|
237
|
+
offset(line, getattr(node, "col_offset", 0)),
|
|
238
|
+
end_line,
|
|
239
|
+
offset(end_line, end_col) if end_line is not None and end_col is not None else None,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
def _diagnose(self, context: _Context, node: ast.AST, code: str, message: str) -> None:
|
|
243
|
+
self.diagnostics.append(
|
|
244
|
+
Diagnostic(
|
|
245
|
+
code,
|
|
246
|
+
"warning",
|
|
247
|
+
message,
|
|
248
|
+
self._location(context, node),
|
|
249
|
+
context.scope,
|
|
250
|
+
"Inspect this boundary; lineage beyond it may be incomplete.",
|
|
251
|
+
)
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
def _new_frame(
|
|
255
|
+
self,
|
|
256
|
+
context: _Context,
|
|
257
|
+
node: ast.AST,
|
|
258
|
+
origin: str,
|
|
259
|
+
evidence: Confidence = "confirmed",
|
|
260
|
+
) -> _State:
|
|
261
|
+
location = self._location(context, node)
|
|
262
|
+
if context.key.startswith("template:"):
|
|
263
|
+
evidence = _confidence(evidence, "inferred")
|
|
264
|
+
record = Frame(
|
|
265
|
+
_identifier("df", location, context.key, origin),
|
|
266
|
+
origin,
|
|
267
|
+
location,
|
|
268
|
+
context.module.name,
|
|
269
|
+
context.scope,
|
|
270
|
+
context.key,
|
|
271
|
+
evidence,
|
|
272
|
+
)
|
|
273
|
+
state = _State(record, index_key=record.id)
|
|
274
|
+
self.frames.append(record)
|
|
275
|
+
self.states[record.id] = state
|
|
276
|
+
if origin.startswith("parameter:"):
|
|
277
|
+
state.history = (
|
|
278
|
+
self._event(
|
|
279
|
+
context,
|
|
280
|
+
node,
|
|
281
|
+
"parameter",
|
|
282
|
+
f"DataFrame parameter {origin.split(':', 1)[1]}",
|
|
283
|
+
frames=(record.id,),
|
|
284
|
+
evidence=evidence,
|
|
285
|
+
completeness="partial",
|
|
286
|
+
),
|
|
287
|
+
)
|
|
288
|
+
return state
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
def _frame_value(state: _State) -> _Value:
|
|
292
|
+
return _Value(
|
|
293
|
+
"frame",
|
|
294
|
+
frame=state,
|
|
295
|
+
evidence=state.record.evidence,
|
|
296
|
+
completeness="complete",
|
|
297
|
+
history=state.history,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
def _operation(
|
|
301
|
+
self,
|
|
302
|
+
context: _Context,
|
|
303
|
+
node: ast.AST,
|
|
304
|
+
kind: str,
|
|
305
|
+
suffix: str,
|
|
306
|
+
evidence: Confidence,
|
|
307
|
+
completeness: Completeness,
|
|
308
|
+
) -> Operation:
|
|
309
|
+
location = self._location(context, node)
|
|
310
|
+
operation = Operation(
|
|
311
|
+
_identifier("op", location, context.key, kind, suffix),
|
|
312
|
+
kind,
|
|
313
|
+
location,
|
|
314
|
+
context.module.name,
|
|
315
|
+
context.scope,
|
|
316
|
+
context.key,
|
|
317
|
+
evidence,
|
|
318
|
+
completeness,
|
|
319
|
+
)
|
|
320
|
+
self.operations.append(operation)
|
|
321
|
+
return operation
|
|
322
|
+
|
|
323
|
+
def _write(
|
|
324
|
+
self,
|
|
325
|
+
state: _State,
|
|
326
|
+
name: str,
|
|
327
|
+
value: _Value,
|
|
328
|
+
context: _Context,
|
|
329
|
+
node: ast.AST,
|
|
330
|
+
kind: str = "derive",
|
|
331
|
+
) -> str:
|
|
332
|
+
value = self._refresh(value)
|
|
333
|
+
previous = state.current.get(name)
|
|
334
|
+
state.positional = None
|
|
335
|
+
if kind == "derive":
|
|
336
|
+
kind = value.producer
|
|
337
|
+
# Reserve the epoch's external version even when that column has not
|
|
338
|
+
# been observed yet: an older lazy copy can materialize it later.
|
|
339
|
+
version = state.versions.get(name, state.epoch) + 1
|
|
340
|
+
state.versions[name] = version
|
|
341
|
+
evidence = _confidence(state.record.evidence, value.evidence)
|
|
342
|
+
if value.kind not in {"literal", "series"}:
|
|
343
|
+
evidence = "unresolved"
|
|
344
|
+
kind = "opaque"
|
|
345
|
+
self._diagnose(
|
|
346
|
+
context,
|
|
347
|
+
node,
|
|
348
|
+
"CT_UNRESOLVED_VALUE",
|
|
349
|
+
"The assigned value has unresolved provenance; no hidden dependencies are assumed.",
|
|
350
|
+
)
|
|
351
|
+
column = Column(
|
|
352
|
+
_identifier("col", state.record.id, name, version),
|
|
353
|
+
state.record.id,
|
|
354
|
+
name,
|
|
355
|
+
version,
|
|
356
|
+
self._location(context, node),
|
|
357
|
+
context.module.name,
|
|
358
|
+
context.scope,
|
|
359
|
+
context.key,
|
|
360
|
+
"write" if kind != "source" else "source",
|
|
361
|
+
evidence,
|
|
362
|
+
value.completeness,
|
|
363
|
+
)
|
|
364
|
+
self.columns[column.id] = column
|
|
365
|
+
state.current[name] = column.id
|
|
366
|
+
state.removed.discard(name)
|
|
367
|
+
if state.schema is not None:
|
|
368
|
+
state.schema.add(name)
|
|
369
|
+
operation = self._operation(context, node, kind, column.id, evidence, value.completeness)
|
|
370
|
+
for dependency in dict.fromkeys(value.dependencies):
|
|
371
|
+
self.edges.append(Edge(dependency, operation.id, "read", evidence))
|
|
372
|
+
self.edges.append(Edge(operation.id, column.id, "write", evidence))
|
|
373
|
+
event_kind = "boundary" if kind == "opaque" else kind
|
|
374
|
+
label = f"{name} = {value.expression or kind}"
|
|
375
|
+
if previous is not None and kind not in {"rename", "source"}:
|
|
376
|
+
event_kind = "overwrite"
|
|
377
|
+
label = f"Overwrite {name}: {value.expression or kind}"
|
|
378
|
+
self.column_history[column.id] = self._event(
|
|
379
|
+
context,
|
|
380
|
+
node,
|
|
381
|
+
event_kind,
|
|
382
|
+
label,
|
|
383
|
+
frames=(state.record.id,),
|
|
384
|
+
inputs=value.dependencies,
|
|
385
|
+
outputs=(column.id,),
|
|
386
|
+
parents=tuple(dict.fromkeys(state.history + self._parents(value))),
|
|
387
|
+
expression=f"{self._code(context, node)} = {value.expression}"
|
|
388
|
+
if value.expression
|
|
389
|
+
else None,
|
|
390
|
+
evidence=evidence,
|
|
391
|
+
completeness=value.completeness,
|
|
392
|
+
)
|
|
393
|
+
return column.id
|
|
394
|
+
|
|
395
|
+
def _external(self, snapshot: _Snapshot, name: str, context: _Context, node: ast.AST) -> str:
|
|
396
|
+
state = snapshot.state
|
|
397
|
+
identifier = _identifier("external", state.record.id, name, snapshot.epoch)
|
|
398
|
+
if identifier not in self.columns:
|
|
399
|
+
evidence = "unresolved" if snapshot.tainted else state.record.evidence
|
|
400
|
+
version = snapshot.versions.get(name, snapshot.epoch) if snapshot.tainted else 0
|
|
401
|
+
state.versions[name] = max(state.versions.get(name, -1), version)
|
|
402
|
+
self.columns[identifier] = Column(
|
|
403
|
+
identifier,
|
|
404
|
+
state.record.id,
|
|
405
|
+
name,
|
|
406
|
+
version,
|
|
407
|
+
self._location(context, node),
|
|
408
|
+
context.module.name,
|
|
409
|
+
context.scope,
|
|
410
|
+
context.key,
|
|
411
|
+
"opaque" if snapshot.tainted else "external_unknown",
|
|
412
|
+
evidence,
|
|
413
|
+
"unresolved" if snapshot.tainted else "partial",
|
|
414
|
+
)
|
|
415
|
+
self.column_history[identifier] = self._event(
|
|
416
|
+
context,
|
|
417
|
+
node,
|
|
418
|
+
"boundary" if snapshot.tainted else "external",
|
|
419
|
+
(
|
|
420
|
+
f"Provenance lost before observing {name}"
|
|
421
|
+
if snapshot.tainted
|
|
422
|
+
else f"Observe input column {name}; definition in external data is unknown"
|
|
423
|
+
),
|
|
424
|
+
frames=(state.record.id,),
|
|
425
|
+
outputs=(identifier,),
|
|
426
|
+
parents=snapshot.history,
|
|
427
|
+
evidence=evidence,
|
|
428
|
+
completeness="unresolved" if snapshot.tainted else "partial",
|
|
429
|
+
)
|
|
430
|
+
snapshot.current[name] = identifier
|
|
431
|
+
return identifier
|
|
432
|
+
|
|
433
|
+
@staticmethod
|
|
434
|
+
def _snapshot(state: _State) -> _Snapshot:
|
|
435
|
+
return _Snapshot(
|
|
436
|
+
state,
|
|
437
|
+
dict(state.current),
|
|
438
|
+
state.fallback,
|
|
439
|
+
None if state.schema is None else set(state.schema),
|
|
440
|
+
state.epoch,
|
|
441
|
+
state.tainted,
|
|
442
|
+
dict(state.versions),
|
|
443
|
+
set(state.removed),
|
|
444
|
+
state.history,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
def _lookup_snapshot(
|
|
448
|
+
self, snapshot: _Snapshot, name: str, context: _Context, node: ast.AST
|
|
449
|
+
) -> str | None:
|
|
450
|
+
if name in snapshot.current:
|
|
451
|
+
return snapshot.current[name]
|
|
452
|
+
if name in snapshot.removed or (
|
|
453
|
+
snapshot.schema is not None and name not in snapshot.schema
|
|
454
|
+
):
|
|
455
|
+
self._diagnose(
|
|
456
|
+
context,
|
|
457
|
+
node,
|
|
458
|
+
"CT_MISSING_COLUMN",
|
|
459
|
+
f"Column {name!r} is absent from the known schema.",
|
|
460
|
+
)
|
|
461
|
+
return None
|
|
462
|
+
if snapshot.fallback is not None:
|
|
463
|
+
copy = snapshot.fallback
|
|
464
|
+
source_name = copy.names.get(name, name)
|
|
465
|
+
source = self._lookup_snapshot(copy.source, source_name, copy.context, copy.node)
|
|
466
|
+
if source is None:
|
|
467
|
+
return None
|
|
468
|
+
evidence = _confidence(copy.evidence, self.columns[source].evidence)
|
|
469
|
+
identifier = _identifier("copy", snapshot.state.record.id, name, snapshot.epoch)
|
|
470
|
+
if identifier not in self.columns:
|
|
471
|
+
record = snapshot.state.record
|
|
472
|
+
version = 0
|
|
473
|
+
snapshot.state.versions[name] = max(snapshot.state.versions.get(name, -1), version)
|
|
474
|
+
column = Column(
|
|
475
|
+
identifier,
|
|
476
|
+
record.id,
|
|
477
|
+
name,
|
|
478
|
+
version,
|
|
479
|
+
self._location(copy.context, copy.node),
|
|
480
|
+
copy.context.module.name,
|
|
481
|
+
copy.context.scope,
|
|
482
|
+
copy.context.key,
|
|
483
|
+
"copy",
|
|
484
|
+
evidence,
|
|
485
|
+
self.columns[source].completeness,
|
|
486
|
+
)
|
|
487
|
+
self.columns[identifier] = column
|
|
488
|
+
operation = self._operation(
|
|
489
|
+
copy.context,
|
|
490
|
+
copy.node,
|
|
491
|
+
"rename" if name in copy.names else "pass",
|
|
492
|
+
identifier,
|
|
493
|
+
evidence,
|
|
494
|
+
column.completeness,
|
|
495
|
+
)
|
|
496
|
+
self.edges.extend(
|
|
497
|
+
[
|
|
498
|
+
Edge(source, operation.id, "read", evidence),
|
|
499
|
+
Edge(operation.id, identifier, "write", evidence),
|
|
500
|
+
]
|
|
501
|
+
)
|
|
502
|
+
kind = "rename" if name in copy.names else record.origin
|
|
503
|
+
self.column_history[identifier] = self._event(
|
|
504
|
+
copy.context,
|
|
505
|
+
copy.node,
|
|
506
|
+
kind,
|
|
507
|
+
f"{source_name} -> {name}"
|
|
508
|
+
if kind == "rename"
|
|
509
|
+
else f"Preserve {name} through {kind}",
|
|
510
|
+
frames=(copy.source.state.record.id, record.id),
|
|
511
|
+
inputs=(source,),
|
|
512
|
+
outputs=(identifier,),
|
|
513
|
+
parents=tuple(
|
|
514
|
+
dict.fromkeys(
|
|
515
|
+
snapshot.history
|
|
516
|
+
+ (
|
|
517
|
+
(self.column_history[source],)
|
|
518
|
+
if source in self.column_history
|
|
519
|
+
else ()
|
|
520
|
+
)
|
|
521
|
+
)
|
|
522
|
+
),
|
|
523
|
+
evidence=evidence,
|
|
524
|
+
completeness=column.completeness,
|
|
525
|
+
)
|
|
526
|
+
snapshot.current[name] = identifier
|
|
527
|
+
return identifier
|
|
528
|
+
return self._external(snapshot, name, context, node)
|
|
529
|
+
|
|
530
|
+
def _read(
|
|
531
|
+
self,
|
|
532
|
+
state: _State,
|
|
533
|
+
name: str,
|
|
534
|
+
context: _Context,
|
|
535
|
+
node: ast.AST,
|
|
536
|
+
*,
|
|
537
|
+
record: bool = True,
|
|
538
|
+
) -> _Value:
|
|
539
|
+
if state.positional is not None and state.schema is None:
|
|
540
|
+
self._diagnose(
|
|
541
|
+
context,
|
|
542
|
+
node,
|
|
543
|
+
"CT_UNSUPPORTED_POSITIONAL_COLUMN",
|
|
544
|
+
"Unlabeled row-result columns require a matching multi-column assignment.",
|
|
545
|
+
)
|
|
546
|
+
return _Value()
|
|
547
|
+
snapshot = self._snapshot(state)
|
|
548
|
+
identifier = self._lookup_snapshot(snapshot, name, context, node)
|
|
549
|
+
if identifier is None:
|
|
550
|
+
return _Value()
|
|
551
|
+
state.current[name] = identifier
|
|
552
|
+
self.consumptions.append(
|
|
553
|
+
Consumption(
|
|
554
|
+
identifier,
|
|
555
|
+
self._location(context, node),
|
|
556
|
+
context.module.name,
|
|
557
|
+
context.scope,
|
|
558
|
+
context.key,
|
|
559
|
+
)
|
|
560
|
+
)
|
|
561
|
+
column = self.columns[identifier]
|
|
562
|
+
parents = (self.column_history[identifier],) if identifier in self.column_history else ()
|
|
563
|
+
if record:
|
|
564
|
+
parents = (
|
|
565
|
+
self._event(
|
|
566
|
+
context,
|
|
567
|
+
node,
|
|
568
|
+
"read",
|
|
569
|
+
f"Read column {name}",
|
|
570
|
+
frames=(state.record.id,),
|
|
571
|
+
inputs=(identifier,),
|
|
572
|
+
parents=parents,
|
|
573
|
+
evidence=column.evidence,
|
|
574
|
+
completeness=column.completeness,
|
|
575
|
+
),
|
|
576
|
+
)
|
|
577
|
+
return _Value(
|
|
578
|
+
"series",
|
|
579
|
+
dependencies=(identifier,),
|
|
580
|
+
evidence=column.evidence,
|
|
581
|
+
completeness=column.completeness,
|
|
582
|
+
epochs=((state.record.id, state.epoch),),
|
|
583
|
+
index_key=state.index_key,
|
|
584
|
+
history=parents,
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
def _refresh(self, value: _Value) -> _Value:
|
|
588
|
+
if any(self.states[identifier].epoch != epoch for identifier, epoch in value.epochs):
|
|
589
|
+
return _Value(
|
|
590
|
+
value.kind,
|
|
591
|
+
evidence="unresolved",
|
|
592
|
+
completeness="unresolved",
|
|
593
|
+
escaped=tuple(
|
|
594
|
+
dict.fromkeys(
|
|
595
|
+
value.escaped + tuple(identifier for identifier, _ in value.epochs)
|
|
596
|
+
)
|
|
597
|
+
),
|
|
598
|
+
)
|
|
599
|
+
return value
|
|
600
|
+
|
|
601
|
+
def _invalidate(
|
|
602
|
+
self,
|
|
603
|
+
values: list[_Value],
|
|
604
|
+
context: _Context | None = None,
|
|
605
|
+
node: ast.AST | None = None,
|
|
606
|
+
) -> None:
|
|
607
|
+
states: dict[str, _State] = {}
|
|
608
|
+
for value in values:
|
|
609
|
+
if value.frame is not None:
|
|
610
|
+
states[value.frame.record.id] = value.frame
|
|
611
|
+
for dependency in value.dependencies:
|
|
612
|
+
frame_id = self.columns[dependency].dataframe_id
|
|
613
|
+
states[frame_id] = self.states[frame_id]
|
|
614
|
+
for frame_id in value.escaped:
|
|
615
|
+
states[frame_id] = self.states[frame_id]
|
|
616
|
+
if states and context is not None and node is not None:
|
|
617
|
+
location = self._location(context, node)
|
|
618
|
+
prior = {event.id: event for event in self.history}
|
|
619
|
+
already = all(
|
|
620
|
+
state.record.id in self.frame_boundaries
|
|
621
|
+
and prior[self.frame_boundaries[state.record.id]].location == location
|
|
622
|
+
and prior[self.frame_boundaries[state.record.id]].context == context.key
|
|
623
|
+
for state in states.values()
|
|
624
|
+
)
|
|
625
|
+
if not already:
|
|
626
|
+
diagnostic = next(
|
|
627
|
+
(
|
|
628
|
+
item
|
|
629
|
+
for item in reversed(self.diagnostics)
|
|
630
|
+
if item.location == location and item.scope == context.scope
|
|
631
|
+
),
|
|
632
|
+
None,
|
|
633
|
+
)
|
|
634
|
+
event = self._event(
|
|
635
|
+
context,
|
|
636
|
+
node,
|
|
637
|
+
"boundary",
|
|
638
|
+
f"Provenance interrupted: {diagnostic.code}"
|
|
639
|
+
if diagnostic
|
|
640
|
+
else "Unresolved mutation effects",
|
|
641
|
+
frames=tuple(states),
|
|
642
|
+
# Forgetting abstract state is an analysis boundary, not
|
|
643
|
+
# evidence that the statement read every known column.
|
|
644
|
+
parents=tuple(
|
|
645
|
+
dict.fromkeys(
|
|
646
|
+
parent for state in states.values() for parent in state.history
|
|
647
|
+
)
|
|
648
|
+
),
|
|
649
|
+
evidence="unresolved",
|
|
650
|
+
completeness="unresolved",
|
|
651
|
+
)
|
|
652
|
+
for frame_id in states:
|
|
653
|
+
self.frame_boundaries[frame_id] = event
|
|
654
|
+
for state in states.values():
|
|
655
|
+
state.current.clear()
|
|
656
|
+
state.fallback = None
|
|
657
|
+
state.schema = None
|
|
658
|
+
state.removed.clear()
|
|
659
|
+
state.epoch += 1
|
|
660
|
+
state.index_key = _identifier("index", state.record.id, state.epoch)
|
|
661
|
+
state.positional = None
|
|
662
|
+
state.tainted = True
|
|
663
|
+
if state.record.id in self.frame_boundaries:
|
|
664
|
+
state.history = (self.frame_boundaries[state.record.id],)
|
|
665
|
+
state.versions = {name: version + 1 for name, version in state.versions.items()}
|
|
666
|
+
|
|
667
|
+
def _escaped(self, values: list[_Value]) -> tuple[str, ...]:
|
|
668
|
+
identifiers: list[str] = []
|
|
669
|
+
for value in values:
|
|
670
|
+
identifiers.extend(value.escaped)
|
|
671
|
+
if value.frame is not None:
|
|
672
|
+
identifiers.append(value.frame.record.id)
|
|
673
|
+
identifiers.extend(self.columns[column].dataframe_id for column in value.dependencies)
|
|
674
|
+
return tuple(dict.fromkeys(identifiers))
|
|
675
|
+
|
|
676
|
+
def _consume(self, value: _Value, context: _Context, node: ast.AST, kind: str) -> str | None:
|
|
677
|
+
dependencies = value.dependencies
|
|
678
|
+
if value.frame is not None:
|
|
679
|
+
dependencies = tuple(value.frame.current.values())
|
|
680
|
+
for identifier in value.escaped:
|
|
681
|
+
dependencies += tuple(self.states[identifier].current.values())
|
|
682
|
+
for identifier in dict.fromkeys(dependencies):
|
|
683
|
+
self.consumptions.append(
|
|
684
|
+
Consumption(
|
|
685
|
+
identifier,
|
|
686
|
+
self._location(context, node),
|
|
687
|
+
context.module.name,
|
|
688
|
+
context.scope,
|
|
689
|
+
context.key,
|
|
690
|
+
kind,
|
|
691
|
+
)
|
|
692
|
+
)
|
|
693
|
+
frames = self._escaped([value])
|
|
694
|
+
if dependencies or frames:
|
|
695
|
+
event_kind = (
|
|
696
|
+
"boundary"
|
|
697
|
+
if kind == "opaque_call"
|
|
698
|
+
else (kind if kind in {"sink", "return", "filter"} else "consume")
|
|
699
|
+
)
|
|
700
|
+
event = self._event(
|
|
701
|
+
context,
|
|
702
|
+
node,
|
|
703
|
+
event_kind,
|
|
704
|
+
kind.replace("_", " "),
|
|
705
|
+
frames=frames,
|
|
706
|
+
inputs=dependencies,
|
|
707
|
+
parents=self._parents(value)
|
|
708
|
+
+ (value.frame.history if value.frame is not None else ()),
|
|
709
|
+
evidence="unresolved" if kind == "opaque_call" else value.evidence,
|
|
710
|
+
completeness="unresolved" if kind == "opaque_call" else value.completeness,
|
|
711
|
+
expression=f"Callback {node.name}(...)"
|
|
712
|
+
if isinstance(node, ast.FunctionDef)
|
|
713
|
+
else None,
|
|
714
|
+
)
|
|
715
|
+
if event_kind == "boundary":
|
|
716
|
+
for frame_id in frames:
|
|
717
|
+
self.frame_boundaries[frame_id] = event
|
|
718
|
+
return event
|
|
719
|
+
return None
|
|
720
|
+
|
|
721
|
+
def _symbol(self, node: ast.AST, context: _Context) -> str:
|
|
722
|
+
if isinstance(node, ast.Name):
|
|
723
|
+
value = context.env.get(node.id, _Value())
|
|
724
|
+
return value.symbol if value.kind == "symbol" else ""
|
|
725
|
+
if isinstance(node, ast.Attribute):
|
|
726
|
+
base = self._symbol(node.value, context)
|
|
727
|
+
symbol = f"{base}.{node.attr}" if base else ""
|
|
728
|
+
if any(
|
|
729
|
+
symbol == changed or symbol.startswith(changed + ".")
|
|
730
|
+
for changed in self.api_mutations
|
|
731
|
+
):
|
|
732
|
+
return ""
|
|
733
|
+
return symbol
|
|
734
|
+
return ""
|
|
735
|
+
|
|
736
|
+
@staticmethod
|
|
737
|
+
def _literal(node: ast.AST) -> bool:
|
|
738
|
+
if isinstance(node, ast.Constant):
|
|
739
|
+
return True
|
|
740
|
+
if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
|
|
741
|
+
return all(_Analyzer._literal(item) for item in node.elts)
|
|
742
|
+
if isinstance(node, ast.Dict):
|
|
743
|
+
return all(key is not None and _Analyzer._literal(key) for key in node.keys) and all(
|
|
744
|
+
_Analyzer._literal(value) for value in node.values
|
|
745
|
+
)
|
|
746
|
+
return (
|
|
747
|
+
isinstance(node, ast.UnaryOp)
|
|
748
|
+
and isinstance(node.op, (ast.UAdd, ast.USub))
|
|
749
|
+
and (_Analyzer._literal(node.operand))
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
@staticmethod
|
|
753
|
+
def _column_target(node: ast.Subscript) -> tuple[ast.expr, str | None]:
|
|
754
|
+
receiver = node.value
|
|
755
|
+
label: ast.AST = node.slice
|
|
756
|
+
if isinstance(receiver, ast.Attribute) and receiver.attr == "loc":
|
|
757
|
+
receiver = receiver.value
|
|
758
|
+
if not isinstance(label, ast.Tuple) or len(label.elts) != 2:
|
|
759
|
+
return receiver, None
|
|
760
|
+
rows, label = label.elts
|
|
761
|
+
if not isinstance(rows, ast.Slice) or any((rows.lower, rows.upper, rows.step)):
|
|
762
|
+
return receiver, None
|
|
763
|
+
if isinstance(label, ast.Constant) and isinstance(label.value, str):
|
|
764
|
+
return receiver, label.value
|
|
765
|
+
return receiver, None
|
|
766
|
+
|
|
767
|
+
@staticmethod
|
|
768
|
+
def _column_names(node: ast.AST) -> tuple[str, ...] | None:
|
|
769
|
+
if isinstance(node, (ast.List, ast.Tuple)) and all(
|
|
770
|
+
isinstance(item, ast.Constant) and isinstance(item.value, str) for item in node.elts
|
|
771
|
+
):
|
|
772
|
+
names = tuple(str(item.value) for item in node.elts if isinstance(item, ast.Constant))
|
|
773
|
+
return names if names and len(names) == len(set(names)) else None
|
|
774
|
+
return None
|
|
775
|
+
|
|
776
|
+
@staticmethod
|
|
777
|
+
def _combined(values: list[_Value], *, kind: Literal["series", "mask"] = "series") -> _Value:
|
|
778
|
+
indexes = {value.index_key for value in values if value.index_key is not None}
|
|
779
|
+
return _Value(
|
|
780
|
+
kind,
|
|
781
|
+
dependencies=tuple(dict.fromkeys(d for value in values for d in value.dependencies)),
|
|
782
|
+
evidence=_confidence(*(value.evidence for value in values)),
|
|
783
|
+
completeness=_completeness(*(value.completeness for value in values)),
|
|
784
|
+
epochs=tuple(dict.fromkeys(epoch for value in values for epoch in value.epochs)),
|
|
785
|
+
index_key=next(iter(indexes)) if len(indexes) == 1 else None,
|
|
786
|
+
history=tuple(dict.fromkeys(item for value in values for item in value.history)),
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
def _materialize(self, state: _State, context: _Context, node: ast.AST) -> None:
|
|
790
|
+
for name in sorted(self._known_columns(self._snapshot(state))):
|
|
791
|
+
self._read(state, name, context, node, record=False)
|
|
792
|
+
|
|
793
|
+
def _method_available(self, owner: str, name: str) -> bool:
|
|
794
|
+
symbol = f"pandas.{owner}.{name}"
|
|
795
|
+
return not any(
|
|
796
|
+
symbol == changed or symbol.startswith(changed + ".") for changed in self.api_mutations
|
|
797
|
+
)
|
|
798
|
+
|
|
799
|
+
def _expr(self, node: ast.AST, context: _Context) -> _Value:
|
|
800
|
+
value = self._expr_value(node, context)
|
|
801
|
+
if not isinstance(node, ast.Name) or not value.expression:
|
|
802
|
+
value = replace(value, expression=self._code(context, node))
|
|
803
|
+
return value
|
|
804
|
+
|
|
805
|
+
def _expr_value(self, node: ast.AST, context: _Context) -> _Value:
|
|
806
|
+
if self._literal(node):
|
|
807
|
+
return _Value("literal", completeness="complete")
|
|
808
|
+
if isinstance(node, ast.Name):
|
|
809
|
+
return self._refresh(context.env.get(node.id, _Value()))
|
|
810
|
+
if isinstance(node, ast.Attribute):
|
|
811
|
+
receiver = self._expr(node.value, context)
|
|
812
|
+
value = self._attribute(receiver, node.attr)
|
|
813
|
+
if context.key.startswith("template:") and receiver.symbol.startswith("module:"):
|
|
814
|
+
return self._isolated(value)
|
|
815
|
+
return value
|
|
816
|
+
if isinstance(node, ast.Subscript):
|
|
817
|
+
receiver_node, name = self._column_target(node)
|
|
818
|
+
receiver = self._expr(receiver_node, context)
|
|
819
|
+
if receiver.frame is not None:
|
|
820
|
+
if name is None:
|
|
821
|
+
if not (isinstance(node.value, ast.Attribute) and node.value.attr == "loc"):
|
|
822
|
+
names = (
|
|
823
|
+
(self._column_names(node.slice) if node.slice.elts else ())
|
|
824
|
+
if isinstance(node.slice, ast.List)
|
|
825
|
+
else None
|
|
826
|
+
)
|
|
827
|
+
if names is not None:
|
|
828
|
+
return self._project(receiver.frame, names, context, node)
|
|
829
|
+
mask = self._expr(node.slice, context)
|
|
830
|
+
if mask.kind == "mask" and mask.index_key == receiver.frame.index_key:
|
|
831
|
+
self._consume(mask, context, node, "filter")
|
|
832
|
+
self._diagnose(
|
|
833
|
+
context,
|
|
834
|
+
node,
|
|
835
|
+
"CT_ROW_CONTROL",
|
|
836
|
+
"Row selection changes control/cardinality; mask columns are "
|
|
837
|
+
"consumed but are not value dependencies of every output column.",
|
|
838
|
+
)
|
|
839
|
+
clone = self._clone(
|
|
840
|
+
receiver.frame, context, node, "filter", {}, receiver.evidence
|
|
841
|
+
)
|
|
842
|
+
clone.index_key = clone.record.id
|
|
843
|
+
self._materialize(clone, context, node)
|
|
844
|
+
return self._frame_value(clone)
|
|
845
|
+
self._diagnose(
|
|
846
|
+
context,
|
|
847
|
+
node,
|
|
848
|
+
"CT_DYNAMIC_COLUMN",
|
|
849
|
+
"Only a literal string column and full-row .loc are supported.",
|
|
850
|
+
)
|
|
851
|
+
return _Value(escaped=(receiver.frame.record.id,))
|
|
852
|
+
return self._read(receiver.frame, name, context, node)
|
|
853
|
+
if receiver.kind != "literal":
|
|
854
|
+
self._diagnose(
|
|
855
|
+
context,
|
|
856
|
+
node,
|
|
857
|
+
"CT_UNRESOLVED_RECEIVER",
|
|
858
|
+
"No resolved Pandas origin or DataFrame annotation for this receiver.",
|
|
859
|
+
)
|
|
860
|
+
return _Value(escaped=receiver.escaped)
|
|
861
|
+
if isinstance(node, ast.Compare):
|
|
862
|
+
values = [self._expr(node.left, context)] + [
|
|
863
|
+
self._expr(item, context) for item in node.comparators
|
|
864
|
+
]
|
|
865
|
+
if (
|
|
866
|
+
all(
|
|
867
|
+
isinstance(op, (ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE))
|
|
868
|
+
for op in node.ops
|
|
869
|
+
)
|
|
870
|
+
and all(value.kind in {"literal", "series"} for value in values)
|
|
871
|
+
and any(value.kind == "series" for value in values)
|
|
872
|
+
):
|
|
873
|
+
return self._combined(values, kind="mask")
|
|
874
|
+
return _Value(escaped=self._escaped(values))
|
|
875
|
+
if isinstance(node, ast.JoinedStr):
|
|
876
|
+
values = []
|
|
877
|
+
for formatted in node.values:
|
|
878
|
+
if isinstance(formatted, ast.FormattedValue):
|
|
879
|
+
values.append(self._expr(formatted.value, context))
|
|
880
|
+
if formatted.format_spec is not None:
|
|
881
|
+
values.append(self._expr(formatted.format_spec, context))
|
|
882
|
+
for value in values:
|
|
883
|
+
self._consume(value, context, node, "observe")
|
|
884
|
+
return replace(self._combined(values), kind="literal", index_key=None)
|
|
885
|
+
if isinstance(node, ast.BinOp) and isinstance(
|
|
886
|
+
node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow)
|
|
887
|
+
):
|
|
888
|
+
left, right = self._expr(node.left, context), self._expr(node.right, context)
|
|
889
|
+
dependencies = tuple(dict.fromkeys(left.dependencies + right.dependencies))
|
|
890
|
+
known = left.kind in {"literal", "series"} and right.kind in {"literal", "series"}
|
|
891
|
+
return _Value(
|
|
892
|
+
"series" if known else "unknown",
|
|
893
|
+
dependencies=dependencies,
|
|
894
|
+
evidence=_confidence(left.evidence, right.evidence) if known else "unresolved",
|
|
895
|
+
completeness=_completeness(left.completeness, right.completeness),
|
|
896
|
+
epochs=tuple(dict.fromkeys(left.epochs + right.epochs)),
|
|
897
|
+
history=tuple(dict.fromkeys(left.history + right.history)),
|
|
898
|
+
)
|
|
899
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
|
|
900
|
+
return self._expr(node.operand, context)
|
|
901
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Invert):
|
|
902
|
+
value = self._expr(node.operand, context)
|
|
903
|
+
return value if value.kind == "mask" else _Value(escaped=self._escaped([value]))
|
|
904
|
+
if isinstance(node, ast.Call):
|
|
905
|
+
return self._call(node, context)
|
|
906
|
+
if isinstance(node, (ast.List, ast.Tuple, ast.Set, ast.Dict)):
|
|
907
|
+
values = [
|
|
908
|
+
self._expr(child, context)
|
|
909
|
+
for child in ast.iter_child_nodes(node)
|
|
910
|
+
if isinstance(child, ast.expr)
|
|
911
|
+
]
|
|
912
|
+
for value in values:
|
|
913
|
+
self._consume(value, context, node, "container")
|
|
914
|
+
return _Value(escaped=self._escaped(values))
|
|
915
|
+
if isinstance(node, ast.Lambda):
|
|
916
|
+
return _Value()
|
|
917
|
+
self._diagnose(
|
|
918
|
+
context,
|
|
919
|
+
node,
|
|
920
|
+
"CT_UNSUPPORTED_EXPRESSION",
|
|
921
|
+
f"Expression {type(node).__name__} is outside the supported arithmetic subset.",
|
|
922
|
+
)
|
|
923
|
+
self._invalidate(list(context.env.values()), context, node)
|
|
924
|
+
return _Value()
|
|
925
|
+
|
|
926
|
+
def _attribute(self, receiver: _Value, name: str) -> _Value:
|
|
927
|
+
if receiver.frame is not None and name == "index":
|
|
928
|
+
return _Value(
|
|
929
|
+
"index",
|
|
930
|
+
index_key=receiver.frame.index_key,
|
|
931
|
+
completeness="complete",
|
|
932
|
+
escaped=(receiver.frame.record.id,),
|
|
933
|
+
)
|
|
934
|
+
if receiver.kind == "series" and name == "str":
|
|
935
|
+
return replace(receiver, kind="accessor")
|
|
936
|
+
if receiver.kind == "symbol":
|
|
937
|
+
symbol = f"{receiver.symbol}.{name}"
|
|
938
|
+
if any(
|
|
939
|
+
symbol == changed or symbol.startswith(changed + ".")
|
|
940
|
+
for changed in self.api_mutations
|
|
941
|
+
):
|
|
942
|
+
return _Value()
|
|
943
|
+
if symbol.startswith("module:"):
|
|
944
|
+
return self._module_member(symbol.removeprefix("module:"))
|
|
945
|
+
return _Value("symbol", symbol=symbol)
|
|
946
|
+
return _Value(escaped=self._escaped([receiver]))
|
|
947
|
+
|
|
948
|
+
def _call(self, node: ast.Call, context: _Context) -> _Value:
|
|
949
|
+
if isinstance(node.func, ast.Attribute):
|
|
950
|
+
receiver = self._expr(node.func.value, context)
|
|
951
|
+
function = self._attribute(receiver, node.func.attr)
|
|
952
|
+
if receiver.frame is not None:
|
|
953
|
+
if node.func.attr == "assign" and self._method_available("DataFrame", "assign"):
|
|
954
|
+
return self._assign(node, receiver.frame, context)
|
|
955
|
+
if node.func.attr == "rename" and self._method_available("DataFrame", "rename"):
|
|
956
|
+
return self._rename(node, receiver.frame, context)
|
|
957
|
+
if node.func.attr == "dropna" and self._method_available("DataFrame", "dropna"):
|
|
958
|
+
return self._dropna(node, receiver.frame, context)
|
|
959
|
+
if node.func.attr in {"to_parquet", "to_csv"} and self._method_available(
|
|
960
|
+
"DataFrame", node.func.attr
|
|
961
|
+
):
|
|
962
|
+
return self._sink(node, receiver, context)
|
|
963
|
+
if receiver.kind == "series":
|
|
964
|
+
if node.func.attr == "apply" and self._method_available("Series", "apply"):
|
|
965
|
+
return self._apply(node, receiver, context)
|
|
966
|
+
if (
|
|
967
|
+
node.func.attr == "tolist"
|
|
968
|
+
and not node.args
|
|
969
|
+
and not node.keywords
|
|
970
|
+
and (
|
|
971
|
+
receiver.components is not None
|
|
972
|
+
and self._method_available("Series", "tolist")
|
|
973
|
+
)
|
|
974
|
+
):
|
|
975
|
+
return replace(receiver, kind="rows")
|
|
976
|
+
if receiver.kind == "accessor" and self._method_available("Series", "str"):
|
|
977
|
+
value = self._string_method(node, receiver, context)
|
|
978
|
+
if value is not None:
|
|
979
|
+
return value
|
|
980
|
+
else:
|
|
981
|
+
receiver = _Value()
|
|
982
|
+
function = self._expr(node.func, context)
|
|
983
|
+
symbol = function.symbol
|
|
984
|
+
arguments = [self._expr(arg, context) for arg in node.args]
|
|
985
|
+
keywords = [(keyword.arg, self._expr(keyword.value, context)) for keyword in node.keywords]
|
|
986
|
+
values = arguments + [value for _, value in keywords]
|
|
987
|
+
if (
|
|
988
|
+
isinstance(node.func, ast.Name)
|
|
989
|
+
and node.func.id not in context.env
|
|
990
|
+
and node.func.id in {"print", "len"}
|
|
991
|
+
and not any(isinstance(arg, ast.Starred) for arg in node.args)
|
|
992
|
+
and not any(name is None for name, _ in keywords)
|
|
993
|
+
):
|
|
994
|
+
valid = (node.func.id == "len" and len(arguments) == 1 and not keywords) or (
|
|
995
|
+
node.func.id == "print"
|
|
996
|
+
and all(
|
|
997
|
+
name in {"sep", "end", "flush"} and value.kind == "literal"
|
|
998
|
+
for name, value in keywords
|
|
999
|
+
)
|
|
1000
|
+
)
|
|
1001
|
+
if valid:
|
|
1002
|
+
for value in values:
|
|
1003
|
+
self._consume(value, context, node, "observe")
|
|
1004
|
+
return _Value(
|
|
1005
|
+
"literal", completeness="partial" if node.func.id == "len" else "complete"
|
|
1006
|
+
)
|
|
1007
|
+
if receiver.frame is not None or receiver.dependencies or receiver.escaped:
|
|
1008
|
+
values.append(receiver)
|
|
1009
|
+
if symbol in {"pandas.read_csv", "pandas.read_parquet", "pandas.DataFrame"}:
|
|
1010
|
+
if symbol == "pandas.DataFrame":
|
|
1011
|
+
rows_frame = self._rows_frame(node, context, arguments, keywords)
|
|
1012
|
+
if rows_frame is not None:
|
|
1013
|
+
return rows_frame
|
|
1014
|
+
return self._source(node, symbol, context, values)
|
|
1015
|
+
if function.function is not None:
|
|
1016
|
+
return self._invoke(function.function, node, context, arguments, keywords)
|
|
1017
|
+
if function.escaped:
|
|
1018
|
+
values.append(function)
|
|
1019
|
+
for value in values:
|
|
1020
|
+
self._consume(value, context, node, "opaque_call")
|
|
1021
|
+
if any(value.frame is not None or value.dependencies or value.escaped for value in values):
|
|
1022
|
+
self._diagnose(
|
|
1023
|
+
context,
|
|
1024
|
+
node,
|
|
1025
|
+
"CT_OPAQUE_CALL",
|
|
1026
|
+
"Call effects and return provenance are unresolved; affected state is invalidated.",
|
|
1027
|
+
)
|
|
1028
|
+
self._invalidate(values, context, node)
|
|
1029
|
+
return _Value(escaped=self._escaped(values))
|
|
1030
|
+
|
|
1031
|
+
def _source(
|
|
1032
|
+
self, node: ast.Call, symbol: str, context: _Context, values: list[_Value]
|
|
1033
|
+
) -> _Value:
|
|
1034
|
+
invalid = any(isinstance(arg, ast.Starred) for arg in node.args) or any(
|
|
1035
|
+
keyword.arg is None for keyword in node.keywords
|
|
1036
|
+
)
|
|
1037
|
+
if symbol == "pandas.read_csv":
|
|
1038
|
+
invalid |= len(node.args) > 1 or any(
|
|
1039
|
+
(
|
|
1040
|
+
kw.arg == "chunksize"
|
|
1041
|
+
and not (isinstance(kw.value, ast.Constant) and kw.value.value is None)
|
|
1042
|
+
)
|
|
1043
|
+
or (
|
|
1044
|
+
kw.arg == "iterator"
|
|
1045
|
+
and not (isinstance(kw.value, ast.Constant) and kw.value.value is False)
|
|
1046
|
+
)
|
|
1047
|
+
for kw in node.keywords
|
|
1048
|
+
)
|
|
1049
|
+
data: ast.AST | None = node.args[0] if len(node.args) == 1 else None
|
|
1050
|
+
if symbol == "pandas.DataFrame":
|
|
1051
|
+
for keyword in node.keywords:
|
|
1052
|
+
if keyword.arg == "data" and not node.args:
|
|
1053
|
+
data = keyword.value
|
|
1054
|
+
invalid |= (
|
|
1055
|
+
len(node.args) > 1
|
|
1056
|
+
or any(kw.arg != "data" for kw in node.keywords)
|
|
1057
|
+
or not isinstance(data, ast.Dict)
|
|
1058
|
+
or not self._literal(data)
|
|
1059
|
+
or any(
|
|
1060
|
+
not isinstance(key, ast.Constant) or not isinstance(key.value, str)
|
|
1061
|
+
for key in data.keys
|
|
1062
|
+
)
|
|
1063
|
+
)
|
|
1064
|
+
if invalid:
|
|
1065
|
+
self._diagnose(
|
|
1066
|
+
context,
|
|
1067
|
+
node,
|
|
1068
|
+
"CT_UNSUPPORTED_SOURCE",
|
|
1069
|
+
"This Pandas constructor/reader signature is outside the supported subset.",
|
|
1070
|
+
)
|
|
1071
|
+
self._invalidate(values, context, node)
|
|
1072
|
+
return _Value()
|
|
1073
|
+
state = self._new_frame(context, node, symbol)
|
|
1074
|
+
resolved = [value.expression for value in values if value.expression]
|
|
1075
|
+
expression = self._code(context, node)
|
|
1076
|
+
if resolved:
|
|
1077
|
+
expression += "\nArguments: " + ", ".join(resolved)
|
|
1078
|
+
state.history = (
|
|
1079
|
+
self._event(
|
|
1080
|
+
context,
|
|
1081
|
+
node,
|
|
1082
|
+
"source",
|
|
1083
|
+
f"Create DataFrame with {symbol}",
|
|
1084
|
+
frames=(state.record.id,),
|
|
1085
|
+
expression=expression,
|
|
1086
|
+
parents=context.parent_history,
|
|
1087
|
+
evidence=state.record.evidence,
|
|
1088
|
+
completeness="complete" if symbol == "pandas.DataFrame" else "partial",
|
|
1089
|
+
),
|
|
1090
|
+
)
|
|
1091
|
+
if isinstance(data, ast.Dict) and symbol == "pandas.DataFrame":
|
|
1092
|
+
state.schema = set()
|
|
1093
|
+
for key in data.keys:
|
|
1094
|
+
assert isinstance(key, ast.Constant) and isinstance(key.value, str)
|
|
1095
|
+
self._write(
|
|
1096
|
+
state,
|
|
1097
|
+
key.value,
|
|
1098
|
+
_Value("literal", completeness="complete"),
|
|
1099
|
+
context,
|
|
1100
|
+
key,
|
|
1101
|
+
"source",
|
|
1102
|
+
)
|
|
1103
|
+
return self._frame_value(state)
|
|
1104
|
+
|
|
1105
|
+
def _apply(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value:
|
|
1106
|
+
arguments = [self._expr(arg, context) for arg in node.args]
|
|
1107
|
+
keywords = [self._expr(keyword.value, context) for keyword in node.keywords]
|
|
1108
|
+
callback = arguments[0].function if len(arguments) == 1 and not node.keywords else None
|
|
1109
|
+
symbols: dict[str, str] = {}
|
|
1110
|
+
if callback is not None:
|
|
1111
|
+
for name, binding in callback.module.env.items():
|
|
1112
|
+
symbol = binding.symbol if binding.kind == "symbol" else "<unresolved>"
|
|
1113
|
+
if any(
|
|
1114
|
+
symbol == changed or symbol.startswith(changed + ".")
|
|
1115
|
+
for changed in self.api_mutations
|
|
1116
|
+
):
|
|
1117
|
+
symbol = "<unresolved>"
|
|
1118
|
+
symbols[name] = symbol
|
|
1119
|
+
summary = summarize_callback(callback.node, symbols=symbols) if callback else None
|
|
1120
|
+
if (
|
|
1121
|
+
summary is None
|
|
1122
|
+
or not summary.supported
|
|
1123
|
+
or summary.may_mutate
|
|
1124
|
+
or summary.components is None
|
|
1125
|
+
):
|
|
1126
|
+
values = [receiver, *arguments, *keywords]
|
|
1127
|
+
if callback is not None and not context.key.startswith("template:"):
|
|
1128
|
+
values.extend(callback.module.env.values())
|
|
1129
|
+
for value in values:
|
|
1130
|
+
self._consume(value, context, node, "opaque_call")
|
|
1131
|
+
self._diagnose(
|
|
1132
|
+
context,
|
|
1133
|
+
node,
|
|
1134
|
+
"CT_UNSUPPORTED_APPLY",
|
|
1135
|
+
"Series.apply requires a resolved scalar callback with supported, fixed-shape "
|
|
1136
|
+
"returns and no unresolved mutation effects.",
|
|
1137
|
+
)
|
|
1138
|
+
self._invalidate(values, context, node)
|
|
1139
|
+
return _Value(escaped=self._escaped(values))
|
|
1140
|
+
assert callback is not None
|
|
1141
|
+
self._consume(receiver, context, node, "apply_input")
|
|
1142
|
+
callback_context = _Context(
|
|
1143
|
+
callback.module,
|
|
1144
|
+
callback.node.name,
|
|
1145
|
+
f"{context.key}>apply:{callback.module.name}.{callback.node.name}",
|
|
1146
|
+
{},
|
|
1147
|
+
context.stack,
|
|
1148
|
+
)
|
|
1149
|
+
self._consume(receiver, callback_context, callback.node, "callback_input")
|
|
1150
|
+
apply_event = self._event(
|
|
1151
|
+
context,
|
|
1152
|
+
node,
|
|
1153
|
+
"apply",
|
|
1154
|
+
f"Apply {callback.module.name}.{callback.node.name}",
|
|
1155
|
+
frames=self._escaped([receiver]),
|
|
1156
|
+
inputs=receiver.dependencies,
|
|
1157
|
+
parents=self._parents(receiver),
|
|
1158
|
+
evidence="inferred",
|
|
1159
|
+
completeness=receiver.completeness,
|
|
1160
|
+
)
|
|
1161
|
+
step_events: dict[str, str] = {}
|
|
1162
|
+
for step in summary.steps:
|
|
1163
|
+
marker = ast.Name(id="callback_step", ctx=ast.Load())
|
|
1164
|
+
marker.lineno, marker.col_offset = step.line, step.column
|
|
1165
|
+
marker.end_lineno, marker.end_col_offset = step.end_line, step.end_column
|
|
1166
|
+
step_events[step.id] = self._event(
|
|
1167
|
+
callback_context,
|
|
1168
|
+
marker,
|
|
1169
|
+
{
|
|
1170
|
+
"value": "callback_step",
|
|
1171
|
+
"control": "callback_control",
|
|
1172
|
+
"configuration": "callback_configuration",
|
|
1173
|
+
"context": "callback_context",
|
|
1174
|
+
}[step.role],
|
|
1175
|
+
step.label,
|
|
1176
|
+
frames=self._escaped([receiver]),
|
|
1177
|
+
inputs=receiver.dependencies if step.depends_on_input else (),
|
|
1178
|
+
parents=tuple(step_events[parent] for parent in step.parents) or (apply_event,),
|
|
1179
|
+
expression=step.expression,
|
|
1180
|
+
evidence="inferred",
|
|
1181
|
+
completeness=receiver.completeness,
|
|
1182
|
+
)
|
|
1183
|
+
self._diagnose(
|
|
1184
|
+
context,
|
|
1185
|
+
node,
|
|
1186
|
+
"CT_CALLBACK_INFERRED",
|
|
1187
|
+
f"Callback {callback.module.name}.{callback.node.name} resolved statically; "
|
|
1188
|
+
"scalar API behavior is inferred and runtime execution is not validated.",
|
|
1189
|
+
)
|
|
1190
|
+
components = tuple(
|
|
1191
|
+
_Value(
|
|
1192
|
+
"series",
|
|
1193
|
+
dependencies=receiver.dependencies if part.depends_on_input else (),
|
|
1194
|
+
evidence=_confidence(receiver.evidence, part.evidence),
|
|
1195
|
+
completeness=_completeness(receiver.completeness, part.completeness),
|
|
1196
|
+
epochs=receiver.epochs,
|
|
1197
|
+
index_key=receiver.index_key,
|
|
1198
|
+
producer="apply",
|
|
1199
|
+
expression=f"{self._code(context, node)} [return component {index + 1}]",
|
|
1200
|
+
history=tuple(step_events[item] for item in part.steps) or (apply_event,),
|
|
1201
|
+
)
|
|
1202
|
+
for index, part in enumerate(summary.components)
|
|
1203
|
+
)
|
|
1204
|
+
result = self._combined(list(components))
|
|
1205
|
+
return replace(result, components=components, producer="apply")
|
|
1206
|
+
|
|
1207
|
+
def _string_method(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value | None:
|
|
1208
|
+
assert isinstance(node.func, ast.Attribute)
|
|
1209
|
+
method = node.func.attr
|
|
1210
|
+
if not self._method_available("Series.str", method):
|
|
1211
|
+
return None
|
|
1212
|
+
no_args = {"strip", "lstrip", "rstrip", "lower", "upper", "casefold", "len"}
|
|
1213
|
+
if method not in no_args | {"contains", "startswith", "endswith", "replace"}:
|
|
1214
|
+
return None
|
|
1215
|
+
if not all(self._literal(arg) for arg in node.args) or not all(
|
|
1216
|
+
keyword.arg is not None and self._literal(keyword.value) for keyword in node.keywords
|
|
1217
|
+
):
|
|
1218
|
+
return None
|
|
1219
|
+
if method in no_args and (node.args or node.keywords):
|
|
1220
|
+
return None
|
|
1221
|
+
self._consume(receiver, context, node, "string_transform")
|
|
1222
|
+
event = self._event(
|
|
1223
|
+
context,
|
|
1224
|
+
node,
|
|
1225
|
+
"string_transform",
|
|
1226
|
+
f"Series.str.{method}",
|
|
1227
|
+
frames=self._escaped([receiver]),
|
|
1228
|
+
inputs=receiver.dependencies,
|
|
1229
|
+
parents=self._parents(receiver),
|
|
1230
|
+
evidence=receiver.evidence,
|
|
1231
|
+
completeness=receiver.completeness,
|
|
1232
|
+
)
|
|
1233
|
+
return replace(
|
|
1234
|
+
receiver,
|
|
1235
|
+
kind="mask" if method in {"contains", "startswith", "endswith"} else "series",
|
|
1236
|
+
components=None,
|
|
1237
|
+
producer="derive",
|
|
1238
|
+
history=(event,),
|
|
1239
|
+
)
|
|
1240
|
+
|
|
1241
|
+
def _rows_frame(
|
|
1242
|
+
self,
|
|
1243
|
+
node: ast.Call,
|
|
1244
|
+
context: _Context,
|
|
1245
|
+
arguments: list[_Value],
|
|
1246
|
+
keywords: list[tuple[str | None, _Value]],
|
|
1247
|
+
) -> _Value | None:
|
|
1248
|
+
options = dict(keywords)
|
|
1249
|
+
data = arguments[0] if len(arguments) == 1 else options.get("data", _Value())
|
|
1250
|
+
if data.kind != "rows":
|
|
1251
|
+
return None
|
|
1252
|
+
index = options.get("index", _Value())
|
|
1253
|
+
valid = (
|
|
1254
|
+
len(arguments) <= 1
|
|
1255
|
+
and not (arguments and "data" in options)
|
|
1256
|
+
and all(name in {"data", "index"} for name, _ in keywords)
|
|
1257
|
+
and data.components is not None
|
|
1258
|
+
and index.kind == "index"
|
|
1259
|
+
and data.index_key is not None
|
|
1260
|
+
and data.index_key == index.index_key
|
|
1261
|
+
)
|
|
1262
|
+
if not valid:
|
|
1263
|
+
self._diagnose(
|
|
1264
|
+
context,
|
|
1265
|
+
node,
|
|
1266
|
+
"CT_UNRESOLVED_ALIGNMENT",
|
|
1267
|
+
"Row-result DataFrame needs fixed width and an explicit matching source index.",
|
|
1268
|
+
)
|
|
1269
|
+
return _Value(escaped=self._escaped([data, index]))
|
|
1270
|
+
state = self._new_frame(context, node, "pandas.DataFrame.rows", data.evidence)
|
|
1271
|
+
state.positional = data.components
|
|
1272
|
+
state.schema = set()
|
|
1273
|
+
assert data.index_key is not None
|
|
1274
|
+
state.index_key = data.index_key
|
|
1275
|
+
return self._frame_value(state)
|
|
1276
|
+
|
|
1277
|
+
def _sink(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value:
|
|
1278
|
+
assert receiver.frame is not None
|
|
1279
|
+
values = [self._expr(arg, context) for arg in node.args]
|
|
1280
|
+
values.extend(self._expr(keyword.value, context) for keyword in node.keywords)
|
|
1281
|
+
valid = len(node.args) <= 1 and all(
|
|
1282
|
+
keyword.arg is not None and self._literal(keyword.value) for keyword in node.keywords
|
|
1283
|
+
)
|
|
1284
|
+
self._materialize(receiver.frame, context, node)
|
|
1285
|
+
self._consume(receiver, context, node, "sink" if valid else "opaque_call")
|
|
1286
|
+
if valid:
|
|
1287
|
+
destinations = [value.expression for value in values if value.expression]
|
|
1288
|
+
self._event(
|
|
1289
|
+
context,
|
|
1290
|
+
node,
|
|
1291
|
+
"sink",
|
|
1292
|
+
"Export DataFrame" + (" to " + destinations[0] if destinations else ""),
|
|
1293
|
+
frames=(receiver.frame.record.id,),
|
|
1294
|
+
inputs=tuple(receiver.frame.current.values()),
|
|
1295
|
+
parents=receiver.frame.history,
|
|
1296
|
+
expression=self._code(context, node)
|
|
1297
|
+
+ ("\nArguments: " + ", ".join(destinations) if destinations else ""),
|
|
1298
|
+
evidence=receiver.evidence,
|
|
1299
|
+
completeness="partial" if receiver.frame.schema is None else "complete",
|
|
1300
|
+
)
|
|
1301
|
+
return _Value("literal", completeness="complete")
|
|
1302
|
+
values.append(receiver)
|
|
1303
|
+
if not context.key.startswith("template:"):
|
|
1304
|
+
for value in list(values):
|
|
1305
|
+
if value.function is not None:
|
|
1306
|
+
values.extend(value.function.module.env.values())
|
|
1307
|
+
self._diagnose(
|
|
1308
|
+
context,
|
|
1309
|
+
node,
|
|
1310
|
+
"CT_UNSUPPORTED_SINK",
|
|
1311
|
+
"Export needs literal keyword options; dynamic options may have unknown effects.",
|
|
1312
|
+
)
|
|
1313
|
+
self._invalidate(values, context, node)
|
|
1314
|
+
return _Value(escaped=self._escaped(values))
|
|
1315
|
+
|
|
1316
|
+
def _project(
|
|
1317
|
+
self, state: _State, names: tuple[str, ...], context: _Context, node: ast.AST
|
|
1318
|
+
) -> _Value:
|
|
1319
|
+
if any(
|
|
1320
|
+
name in state.removed or (state.schema is not None and name not in state.schema)
|
|
1321
|
+
for name in names
|
|
1322
|
+
):
|
|
1323
|
+
self._diagnose(
|
|
1324
|
+
context, node, "CT_MISSING_COLUMN", "Projection contains a missing column."
|
|
1325
|
+
)
|
|
1326
|
+
return _Value()
|
|
1327
|
+
clone = self._clone(state, context, node, "projection", {}, state.record.evidence)
|
|
1328
|
+
clone.schema = set(names)
|
|
1329
|
+
# Record the structural selection even when no retained column creates
|
|
1330
|
+
# a pass event. An empty projection has a meaningful column fate.
|
|
1331
|
+
projection = self._event(
|
|
1332
|
+
context,
|
|
1333
|
+
node,
|
|
1334
|
+
"projection",
|
|
1335
|
+
"Project literal columns",
|
|
1336
|
+
frames=(state.record.id, clone.record.id),
|
|
1337
|
+
parents=state.history,
|
|
1338
|
+
evidence=state.record.evidence,
|
|
1339
|
+
completeness="complete",
|
|
1340
|
+
)
|
|
1341
|
+
clone.history = (projection,)
|
|
1342
|
+
for name in names:
|
|
1343
|
+
self._read(clone, name, context, node, record=False)
|
|
1344
|
+
return self._frame_value(clone)
|
|
1345
|
+
|
|
1346
|
+
def _dropna(self, node: ast.Call, state: _State, context: _Context) -> _Value:
|
|
1347
|
+
valid = not node.args
|
|
1348
|
+
subset: tuple[str, ...] | None = None
|
|
1349
|
+
inplace = False
|
|
1350
|
+
for keyword in node.keywords:
|
|
1351
|
+
if keyword.arg == "subset":
|
|
1352
|
+
subset = self._column_names(keyword.value)
|
|
1353
|
+
valid &= subset is not None
|
|
1354
|
+
elif keyword.arg == "inplace" and isinstance(keyword.value, ast.Constant):
|
|
1355
|
+
valid &= type(keyword.value.value) is bool
|
|
1356
|
+
inplace = keyword.value.value is True
|
|
1357
|
+
elif keyword.arg == "axis" and isinstance(keyword.value, ast.Constant):
|
|
1358
|
+
valid &= keyword.value.value in {0, "index"}
|
|
1359
|
+
elif keyword.arg == "how" and isinstance(keyword.value, ast.Constant):
|
|
1360
|
+
valid &= keyword.value.value in {"any", "all"}
|
|
1361
|
+
else:
|
|
1362
|
+
valid = False
|
|
1363
|
+
if not valid:
|
|
1364
|
+
for arg in node.args:
|
|
1365
|
+
self._expr(arg, context)
|
|
1366
|
+
for keyword in node.keywords:
|
|
1367
|
+
self._expr(keyword.value, context)
|
|
1368
|
+
self._diagnose(
|
|
1369
|
+
context,
|
|
1370
|
+
node,
|
|
1371
|
+
"CT_UNSUPPORTED_DROPNA",
|
|
1372
|
+
"Only row dropna with literal options is supported.",
|
|
1373
|
+
)
|
|
1374
|
+
self._invalidate([self._frame_value(state)], context, node)
|
|
1375
|
+
return _Value()
|
|
1376
|
+
for name in subset or sorted(self._known_columns(self._snapshot(state))):
|
|
1377
|
+
self._consume(
|
|
1378
|
+
self._read(state, name, context, node, record=False), context, node, "filter"
|
|
1379
|
+
)
|
|
1380
|
+
self._diagnose(
|
|
1381
|
+
context,
|
|
1382
|
+
node,
|
|
1383
|
+
"CT_ROW_CONTROL",
|
|
1384
|
+
"Dropna changes row control/cardinality; subset columns are recorded as filter "
|
|
1385
|
+
"consumptions without all-to-all value dependencies.",
|
|
1386
|
+
)
|
|
1387
|
+
if inplace:
|
|
1388
|
+
state.index_key = _identifier("index", state.index_key, self._location(context, node))
|
|
1389
|
+
return _Value("literal", completeness="complete")
|
|
1390
|
+
clone = self._clone(state, context, node, "dropna", {}, state.record.evidence)
|
|
1391
|
+
clone.index_key = clone.record.id
|
|
1392
|
+
self._materialize(clone, context, node)
|
|
1393
|
+
return self._frame_value(clone)
|
|
1394
|
+
|
|
1395
|
+
def _clone(
|
|
1396
|
+
self,
|
|
1397
|
+
state: _State,
|
|
1398
|
+
context: _Context,
|
|
1399
|
+
node: ast.AST,
|
|
1400
|
+
kind: str,
|
|
1401
|
+
names: dict[str, str],
|
|
1402
|
+
evidence: Confidence,
|
|
1403
|
+
) -> _State:
|
|
1404
|
+
clone = self._new_frame(context, node, kind, _confidence(state.record.evidence, evidence))
|
|
1405
|
+
clone.schema = (
|
|
1406
|
+
None
|
|
1407
|
+
if state.schema is None
|
|
1408
|
+
else {
|
|
1409
|
+
next((new for new, old in names.items() if old == name), name)
|
|
1410
|
+
for name in state.schema
|
|
1411
|
+
}
|
|
1412
|
+
)
|
|
1413
|
+
clone.fallback = _Copy(self._snapshot(state), names, context, node, clone.record.evidence)
|
|
1414
|
+
clone.history = state.history
|
|
1415
|
+
clone.index_key = state.index_key
|
|
1416
|
+
clone.removed = (state.removed | set(names.values())) - names.keys()
|
|
1417
|
+
return clone
|
|
1418
|
+
|
|
1419
|
+
def _known_columns(self, snapshot: _Snapshot) -> set[str]:
|
|
1420
|
+
known = set(snapshot.current)
|
|
1421
|
+
if snapshot.fallback is not None:
|
|
1422
|
+
names = snapshot.fallback.names
|
|
1423
|
+
known.update(
|
|
1424
|
+
next((new for new, old in names.items() if old == name), name)
|
|
1425
|
+
for name in self._known_columns(snapshot.fallback.source)
|
|
1426
|
+
)
|
|
1427
|
+
known -= snapshot.removed
|
|
1428
|
+
return known if snapshot.schema is None else known & snapshot.schema
|
|
1429
|
+
|
|
1430
|
+
def _rename(self, node: ast.Call, state: _State, context: _Context) -> _Value:
|
|
1431
|
+
mapping: dict[str, str] = {}
|
|
1432
|
+
valid = not node.args
|
|
1433
|
+
found = False
|
|
1434
|
+
inplace = False
|
|
1435
|
+
for keyword in node.keywords:
|
|
1436
|
+
if keyword.arg == "columns" and isinstance(keyword.value, ast.Dict):
|
|
1437
|
+
found = True
|
|
1438
|
+
for key, value in zip(keyword.value.keys, keyword.value.values, strict=True):
|
|
1439
|
+
if (
|
|
1440
|
+
isinstance(key, ast.Constant)
|
|
1441
|
+
and isinstance(key.value, str)
|
|
1442
|
+
and isinstance(value, ast.Constant)
|
|
1443
|
+
and isinstance(value.value, str)
|
|
1444
|
+
):
|
|
1445
|
+
mapping[key.value] = value.value
|
|
1446
|
+
else:
|
|
1447
|
+
valid = False
|
|
1448
|
+
elif keyword.arg == "inplace" and isinstance(keyword.value, ast.Constant):
|
|
1449
|
+
valid &= type(keyword.value.value) is bool
|
|
1450
|
+
inplace = keyword.value.value is True
|
|
1451
|
+
else:
|
|
1452
|
+
valid = False
|
|
1453
|
+
valid &= found and len(set(mapping.values())) == len(mapping)
|
|
1454
|
+
known = self._known_columns(self._snapshot(state))
|
|
1455
|
+
valid &= not any(new in known - mapping.keys() for new in mapping.values())
|
|
1456
|
+
valid &= not any(old in state.removed for old in mapping)
|
|
1457
|
+
if state.schema is not None:
|
|
1458
|
+
valid &= all(name in state.schema for name in mapping)
|
|
1459
|
+
valid &= not any(new in state.schema - mapping.keys() for new in mapping.values())
|
|
1460
|
+
if not valid:
|
|
1461
|
+
for arg in node.args:
|
|
1462
|
+
self._expr(arg, context)
|
|
1463
|
+
for keyword in node.keywords:
|
|
1464
|
+
self._expr(keyword.value, context)
|
|
1465
|
+
self._diagnose(
|
|
1466
|
+
context,
|
|
1467
|
+
node,
|
|
1468
|
+
"CT_UNSUPPORTED_RENAME",
|
|
1469
|
+
"Rename needs an injective literal columns mapping, no known schema collision, "
|
|
1470
|
+
"and a literal boolean inplace option.",
|
|
1471
|
+
)
|
|
1472
|
+
if any(keyword.arg == "inplace" for keyword in node.keywords):
|
|
1473
|
+
self._invalidate([self._frame_value(state)], context, node)
|
|
1474
|
+
return _Value()
|
|
1475
|
+
evidence: Confidence = "confirmed" if state.schema is not None else "inferred"
|
|
1476
|
+
if evidence == "inferred":
|
|
1477
|
+
self._diagnose(
|
|
1478
|
+
context,
|
|
1479
|
+
node,
|
|
1480
|
+
"CT_RENAME_SCHEMA_ASSUMPTION",
|
|
1481
|
+
"Rename assumes source columns exist and destination labels do not collide; "
|
|
1482
|
+
"the external schema is unknown.",
|
|
1483
|
+
)
|
|
1484
|
+
if inplace:
|
|
1485
|
+
before = self._snapshot(state)
|
|
1486
|
+
values: dict[str, _Value] = {}
|
|
1487
|
+
for old, new in mapping.items():
|
|
1488
|
+
source = self._lookup_snapshot(before, old, context, node)
|
|
1489
|
+
if source is not None:
|
|
1490
|
+
column = self.columns[source]
|
|
1491
|
+
values[new] = _Value(
|
|
1492
|
+
"series",
|
|
1493
|
+
dependencies=(source,),
|
|
1494
|
+
evidence=_confidence(evidence, column.evidence),
|
|
1495
|
+
completeness=column.completeness,
|
|
1496
|
+
)
|
|
1497
|
+
for old in mapping:
|
|
1498
|
+
state.current.pop(old, None)
|
|
1499
|
+
state.removed = (state.removed | set(mapping)) - set(mapping.values())
|
|
1500
|
+
if state.schema is not None:
|
|
1501
|
+
state.schema = {mapping.get(name, name) for name in state.schema}
|
|
1502
|
+
for new, column_value in values.items():
|
|
1503
|
+
self._write(state, new, column_value, context, node, "rename")
|
|
1504
|
+
return _Value("literal", completeness="complete")
|
|
1505
|
+
clone = self._clone(
|
|
1506
|
+
state, context, node, "rename", {new: old for old, new in mapping.items()}, evidence
|
|
1507
|
+
)
|
|
1508
|
+
for name in mapping.values():
|
|
1509
|
+
self._read(clone, name, context, node, record=False)
|
|
1510
|
+
return self._frame_value(clone)
|
|
1511
|
+
|
|
1512
|
+
def _assign(self, node: ast.Call, state: _State, context: _Context) -> _Value:
|
|
1513
|
+
if node.args or any(keyword.arg is None for keyword in node.keywords):
|
|
1514
|
+
self._diagnose(
|
|
1515
|
+
context,
|
|
1516
|
+
node,
|
|
1517
|
+
"CT_UNSUPPORTED_ASSIGN",
|
|
1518
|
+
"Assign requires explicit keyword arguments.",
|
|
1519
|
+
)
|
|
1520
|
+
self._invalidate([self._frame_value(state)], context, node)
|
|
1521
|
+
return _Value()
|
|
1522
|
+
eager: dict[str, _Value] = {}
|
|
1523
|
+
for keyword in node.keywords:
|
|
1524
|
+
assert keyword.arg is not None
|
|
1525
|
+
if not isinstance(keyword.value, ast.Lambda):
|
|
1526
|
+
eager[keyword.arg] = self._expr(keyword.value, context)
|
|
1527
|
+
clone = self._clone(state, context, node, "assign", {}, state.record.evidence)
|
|
1528
|
+
for keyword in node.keywords:
|
|
1529
|
+
assert keyword.arg is not None
|
|
1530
|
+
if isinstance(keyword.value, ast.Lambda):
|
|
1531
|
+
function = keyword.value
|
|
1532
|
+
args = function.args
|
|
1533
|
+
if (
|
|
1534
|
+
len(args.posonlyargs + args.args) != 1
|
|
1535
|
+
or args.kwonlyargs
|
|
1536
|
+
or args.vararg
|
|
1537
|
+
or args.kwarg
|
|
1538
|
+
or args.defaults
|
|
1539
|
+
):
|
|
1540
|
+
self._diagnose(
|
|
1541
|
+
context,
|
|
1542
|
+
function,
|
|
1543
|
+
"CT_UNSUPPORTED_ASSIGN",
|
|
1544
|
+
"Assign lambdas need one positional parameter and no defaults.",
|
|
1545
|
+
)
|
|
1546
|
+
value = _Value()
|
|
1547
|
+
else:
|
|
1548
|
+
env = dict(context.env)
|
|
1549
|
+
env[(args.posonlyargs + args.args)[0].arg] = self._frame_value(clone)
|
|
1550
|
+
nested = _Context(
|
|
1551
|
+
context.module, context.scope, context.key, env, context.stack
|
|
1552
|
+
)
|
|
1553
|
+
value = self._expr(function.body, nested)
|
|
1554
|
+
else:
|
|
1555
|
+
value = eager[keyword.arg]
|
|
1556
|
+
if value.kind not in {"literal", "series"}:
|
|
1557
|
+
self._diagnose(
|
|
1558
|
+
context,
|
|
1559
|
+
keyword.value,
|
|
1560
|
+
"CT_UNSUPPORTED_ASSIGN",
|
|
1561
|
+
"An unresolved assign value may be callable; its effects on the copy "
|
|
1562
|
+
"are not modeled.",
|
|
1563
|
+
)
|
|
1564
|
+
self._invalidate([self._frame_value(clone)], context, keyword.value)
|
|
1565
|
+
self._write(clone, keyword.arg, value, context, keyword.value)
|
|
1566
|
+
return self._frame_value(clone)
|
|
1567
|
+
|
|
1568
|
+
def _module_member(self, dotted: str, *, prefer_attribute: bool = True) -> _Value:
|
|
1569
|
+
if self._ambiguous_module(dotted):
|
|
1570
|
+
return _Value()
|
|
1571
|
+
module_name, _, member = dotted.rpartition(".")
|
|
1572
|
+
parent = self.modules.get(module_name)
|
|
1573
|
+
if parent is not None:
|
|
1574
|
+
self._module(parent)
|
|
1575
|
+
if prefer_attribute and member in parent.env:
|
|
1576
|
+
return parent.env[member]
|
|
1577
|
+
if dotted in self.modules:
|
|
1578
|
+
self._module(self.modules[dotted])
|
|
1579
|
+
value = _Value("symbol", symbol=f"module:{dotted}")
|
|
1580
|
+
if parent is not None:
|
|
1581
|
+
parent.env[member] = value
|
|
1582
|
+
return value
|
|
1583
|
+
if parent is not None:
|
|
1584
|
+
return parent.env.get(member, _Value())
|
|
1585
|
+
return _Value("symbol", symbol=f"module:{dotted}")
|
|
1586
|
+
|
|
1587
|
+
def _ambiguous_module(self, name: str) -> bool:
|
|
1588
|
+
return any(name == module or name.startswith(module + ".") for module in self.ambiguous)
|
|
1589
|
+
|
|
1590
|
+
def _import(self, node: ast.Import | ast.ImportFrom, context: _Context) -> None:
|
|
1591
|
+
if isinstance(node, ast.Import):
|
|
1592
|
+
for alias in node.names:
|
|
1593
|
+
name = alias.asname or alias.name.split(".")[0]
|
|
1594
|
+
if self._ambiguous_module(alias.name):
|
|
1595
|
+
context.env[name] = _Value()
|
|
1596
|
+
continue
|
|
1597
|
+
if alias.name in self.modules or any(
|
|
1598
|
+
key.startswith(alias.name + ".") for key in self.modules
|
|
1599
|
+
):
|
|
1600
|
+
self._module_member(alias.name, prefer_attribute=False)
|
|
1601
|
+
module_name = alias.name if alias.asname else alias.name.split(".")[0]
|
|
1602
|
+
context.env[name] = _Value("symbol", symbol=f"module:{module_name}")
|
|
1603
|
+
else:
|
|
1604
|
+
context.env[name] = _Value("symbol", symbol=alias.name)
|
|
1605
|
+
return
|
|
1606
|
+
module_name = node.module or ""
|
|
1607
|
+
if node.level:
|
|
1608
|
+
package = context.module.name.split(".")
|
|
1609
|
+
if not context.module.path.endswith("/__init__.py"):
|
|
1610
|
+
package = package[:-1]
|
|
1611
|
+
if node.level > len(package):
|
|
1612
|
+
module_name = ""
|
|
1613
|
+
else:
|
|
1614
|
+
module_name = ".".join(
|
|
1615
|
+
package[: len(package) - node.level + 1]
|
|
1616
|
+
+ (module_name.split(".") if module_name else [])
|
|
1617
|
+
)
|
|
1618
|
+
for alias in node.names:
|
|
1619
|
+
if alias.name == "*":
|
|
1620
|
+
self._diagnose(
|
|
1621
|
+
context, node, "CT_STAR_IMPORT", "Wildcard imports invalidate name resolution."
|
|
1622
|
+
)
|
|
1623
|
+
self._invalidate(list(context.env.values()), context, node)
|
|
1624
|
+
context.env.clear()
|
|
1625
|
+
continue
|
|
1626
|
+
if self._ambiguous_module(module_name):
|
|
1627
|
+
context.env[alias.asname or alias.name] = _Value()
|
|
1628
|
+
continue
|
|
1629
|
+
if module_name in self.modules or any(
|
|
1630
|
+
key.startswith(module_name + ".") for key in self.modules
|
|
1631
|
+
):
|
|
1632
|
+
value = self._module_member(f"{module_name}.{alias.name}")
|
|
1633
|
+
else:
|
|
1634
|
+
value = _Value("symbol", symbol=f"{module_name}.{alias.name}")
|
|
1635
|
+
if value.symbol in self.api_mutations:
|
|
1636
|
+
value = _Value()
|
|
1637
|
+
if context.key.startswith("template:"):
|
|
1638
|
+
value = self._isolated(value)
|
|
1639
|
+
context.env[alias.asname or alias.name] = value
|
|
1640
|
+
|
|
1641
|
+
def _bind(self, target: ast.AST, value: _Value, context: _Context) -> None:
|
|
1642
|
+
if isinstance(target, ast.Name):
|
|
1643
|
+
context.env[target.id] = value
|
|
1644
|
+
if value.frame is not None:
|
|
1645
|
+
frame = value.frame
|
|
1646
|
+
event = self._event(
|
|
1647
|
+
context,
|
|
1648
|
+
target,
|
|
1649
|
+
"bind",
|
|
1650
|
+
f"{target.id} <- {value.expression or frame.record.origin}",
|
|
1651
|
+
frames=(frame.record.id,),
|
|
1652
|
+
parents=tuple(dict.fromkeys(frame.history + value.history)),
|
|
1653
|
+
expression=f"{target.id} = {value.expression}"
|
|
1654
|
+
if value.expression
|
|
1655
|
+
else target.id,
|
|
1656
|
+
evidence=value.evidence,
|
|
1657
|
+
completeness=value.completeness,
|
|
1658
|
+
)
|
|
1659
|
+
frame.history = (event,)
|
|
1660
|
+
context.env[target.id] = replace(value, history=(event,))
|
|
1661
|
+
return
|
|
1662
|
+
if isinstance(target, ast.Subscript):
|
|
1663
|
+
receiver_node, name = self._column_target(target)
|
|
1664
|
+
receiver = self._expr(receiver_node, context)
|
|
1665
|
+
if receiver.frame is not None:
|
|
1666
|
+
if name is not None:
|
|
1667
|
+
self._write(receiver.frame, name, value, context, target)
|
|
1668
|
+
else:
|
|
1669
|
+
names = (
|
|
1670
|
+
self._column_names(target.slice)
|
|
1671
|
+
if isinstance(target.slice, ast.List)
|
|
1672
|
+
else None
|
|
1673
|
+
)
|
|
1674
|
+
if names is not None:
|
|
1675
|
+
source = value.frame
|
|
1676
|
+
parts = source.positional if source is not None else None
|
|
1677
|
+
if (
|
|
1678
|
+
parts is not None
|
|
1679
|
+
and len(parts) == len(names)
|
|
1680
|
+
and source is not None
|
|
1681
|
+
and source.index_key == receiver.frame.index_key
|
|
1682
|
+
):
|
|
1683
|
+
for column_name, part in zip(names, parts, strict=True):
|
|
1684
|
+
self._write(receiver.frame, column_name, part, context, target)
|
|
1685
|
+
return
|
|
1686
|
+
self._diagnose(
|
|
1687
|
+
context,
|
|
1688
|
+
target,
|
|
1689
|
+
"CT_UNRESOLVED_MULTI_ASSIGN",
|
|
1690
|
+
"Multi-column assignment needs matching tuple width and row index; "
|
|
1691
|
+
"no positional dependencies are assumed.",
|
|
1692
|
+
)
|
|
1693
|
+
self._invalidate([receiver], context, target)
|
|
1694
|
+
return
|
|
1695
|
+
self._diagnose(
|
|
1696
|
+
context,
|
|
1697
|
+
target,
|
|
1698
|
+
"CT_DYNAMIC_COLUMN",
|
|
1699
|
+
"A dynamic or partial column write invalidates the DataFrame state.",
|
|
1700
|
+
)
|
|
1701
|
+
self._invalidate([receiver], context, target)
|
|
1702
|
+
elif receiver.kind != "literal":
|
|
1703
|
+
self._diagnose(
|
|
1704
|
+
context,
|
|
1705
|
+
target,
|
|
1706
|
+
"CT_UNRESOLVED_RECEIVER",
|
|
1707
|
+
"Cannot bind a column write to a resolved DataFrame.",
|
|
1708
|
+
)
|
|
1709
|
+
self._invalidate([receiver], context, target)
|
|
1710
|
+
return
|
|
1711
|
+
if isinstance(target, ast.Attribute):
|
|
1712
|
+
receiver = self._expr(target.value, context)
|
|
1713
|
+
symbol = self._symbol(target, context)
|
|
1714
|
+
if symbol:
|
|
1715
|
+
self.api_mutations.add(symbol)
|
|
1716
|
+
if receiver.frame is not None or receiver.dependencies or symbol.startswith("pandas"):
|
|
1717
|
+
self._diagnose(
|
|
1718
|
+
context,
|
|
1719
|
+
target,
|
|
1720
|
+
"CT_ATTRIBUTE_MUTATION",
|
|
1721
|
+
"Attribute mutation is not modeled and invalidates affected evidence.",
|
|
1722
|
+
)
|
|
1723
|
+
self._invalidate([receiver], context, target)
|
|
1724
|
+
return
|
|
1725
|
+
self._diagnose(
|
|
1726
|
+
context, target, "CT_UNSUPPORTED_TARGET", "Unpacking/chained targets are not modeled."
|
|
1727
|
+
)
|
|
1728
|
+
self._invalidate([value] + list(context.env.values()), context, target)
|
|
1729
|
+
for child in ast.walk(target):
|
|
1730
|
+
if isinstance(child, ast.Name):
|
|
1731
|
+
context.env[child.id] = _Value()
|
|
1732
|
+
|
|
1733
|
+
@staticmethod
|
|
1734
|
+
def _bound_names(node: ast.AST) -> set[str]:
|
|
1735
|
+
names: set[str] = set()
|
|
1736
|
+
|
|
1737
|
+
class Visitor(ast.NodeVisitor):
|
|
1738
|
+
def visit_Name(self, child: ast.Name) -> None:
|
|
1739
|
+
if isinstance(child.ctx, (ast.Store, ast.Del)):
|
|
1740
|
+
names.add(child.id)
|
|
1741
|
+
|
|
1742
|
+
def visit_FunctionDef(self, child: ast.FunctionDef) -> None:
|
|
1743
|
+
names.add(child.name)
|
|
1744
|
+
|
|
1745
|
+
def visit_AsyncFunctionDef(self, child: ast.AsyncFunctionDef) -> None:
|
|
1746
|
+
names.add(child.name)
|
|
1747
|
+
|
|
1748
|
+
def visit_ClassDef(self, child: ast.ClassDef) -> None:
|
|
1749
|
+
names.add(child.name)
|
|
1750
|
+
|
|
1751
|
+
def visit_Lambda(self, child: ast.Lambda) -> None:
|
|
1752
|
+
pass
|
|
1753
|
+
|
|
1754
|
+
def visit_Import(self, child: ast.Import) -> None:
|
|
1755
|
+
names.update(alias.asname or alias.name.split(".")[0] for alias in child.names)
|
|
1756
|
+
|
|
1757
|
+
def visit_ImportFrom(self, child: ast.ImportFrom) -> None:
|
|
1758
|
+
names.update(alias.asname or alias.name for alias in child.names)
|
|
1759
|
+
|
|
1760
|
+
def visit_ExceptHandler(self, child: ast.ExceptHandler) -> None:
|
|
1761
|
+
if child.name:
|
|
1762
|
+
names.add(child.name)
|
|
1763
|
+
self.generic_visit(child)
|
|
1764
|
+
|
|
1765
|
+
def visit_MatchAs(self, child: ast.MatchAs) -> None:
|
|
1766
|
+
if child.name:
|
|
1767
|
+
names.add(child.name)
|
|
1768
|
+
self.generic_visit(child)
|
|
1769
|
+
|
|
1770
|
+
def visit_MatchStar(self, child: ast.MatchStar) -> None:
|
|
1771
|
+
if child.name:
|
|
1772
|
+
names.add(child.name)
|
|
1773
|
+
|
|
1774
|
+
def visit_MatchMapping(self, child: ast.MatchMapping) -> None:
|
|
1775
|
+
if child.rest:
|
|
1776
|
+
names.add(child.rest)
|
|
1777
|
+
self.generic_visit(child)
|
|
1778
|
+
|
|
1779
|
+
visitor = Visitor()
|
|
1780
|
+
if isinstance(node, ast.FunctionDef):
|
|
1781
|
+
for statement in node.body:
|
|
1782
|
+
visitor.visit(statement)
|
|
1783
|
+
else:
|
|
1784
|
+
visitor.visit(node)
|
|
1785
|
+
return names
|
|
1786
|
+
|
|
1787
|
+
def _statements(self, statements: list[ast.stmt], context: _Context) -> _Value:
|
|
1788
|
+
for node in statements:
|
|
1789
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
1790
|
+
self._import(node, context)
|
|
1791
|
+
elif isinstance(node, ast.Assign):
|
|
1792
|
+
value = self._expr(node.value, context)
|
|
1793
|
+
for target in node.targets:
|
|
1794
|
+
self._bind(target, value, context)
|
|
1795
|
+
elif isinstance(node, ast.AnnAssign):
|
|
1796
|
+
if node.value is not None:
|
|
1797
|
+
self._bind(node.target, self._expr(node.value, context), context)
|
|
1798
|
+
elif isinstance(node, ast.Expr):
|
|
1799
|
+
self._expr(node.value, context)
|
|
1800
|
+
elif isinstance(node, ast.FunctionDef):
|
|
1801
|
+
for decorator in node.decorator_list:
|
|
1802
|
+
decoration = self._expr(decorator, context)
|
|
1803
|
+
self._invalidate([decoration], context, decorator)
|
|
1804
|
+
if decoration.function is not None and not context.key.startswith("template:"):
|
|
1805
|
+
self._diagnose(
|
|
1806
|
+
context,
|
|
1807
|
+
decorator,
|
|
1808
|
+
"CT_DECORATOR_EFFECTS",
|
|
1809
|
+
"Applying the resolved decorator may mutate module state; its globals "
|
|
1810
|
+
"are invalidated rather than assuming a pure decoration.",
|
|
1811
|
+
)
|
|
1812
|
+
self._invalidate(
|
|
1813
|
+
list(decoration.function.module.env.values()), context, decorator
|
|
1814
|
+
)
|
|
1815
|
+
arguments = node.args.posonlyargs + node.args.args
|
|
1816
|
+
default_nodes = list(
|
|
1817
|
+
zip(arguments[-len(node.args.defaults) :], node.args.defaults, strict=False)
|
|
1818
|
+
)
|
|
1819
|
+
defaults = {arg.arg: self._expr(default, context) for arg, default in default_nodes}
|
|
1820
|
+
defaults.update(
|
|
1821
|
+
{
|
|
1822
|
+
arg.arg: self._expr(default, context)
|
|
1823
|
+
for arg, default in zip(
|
|
1824
|
+
node.args.kwonlyargs, node.args.kw_defaults, strict=True
|
|
1825
|
+
)
|
|
1826
|
+
if default is not None
|
|
1827
|
+
}
|
|
1828
|
+
)
|
|
1829
|
+
annotations = [
|
|
1830
|
+
argument.annotation for argument in arguments + node.args.kwonlyargs
|
|
1831
|
+
] + [node.returns]
|
|
1832
|
+
if node.args.vararg:
|
|
1833
|
+
annotations.append(node.args.vararg.annotation)
|
|
1834
|
+
if node.args.kwarg:
|
|
1835
|
+
annotations.append(node.args.kwarg.annotation)
|
|
1836
|
+
for annotation in annotations:
|
|
1837
|
+
if annotation is not None and not isinstance(
|
|
1838
|
+
annotation, (ast.Name, ast.Attribute, ast.Constant)
|
|
1839
|
+
):
|
|
1840
|
+
# Annotation evaluation differs across Python versions
|
|
1841
|
+
# and future imports. Never execute it, and do not let
|
|
1842
|
+
# possible effects preserve confirmed object history.
|
|
1843
|
+
self._diagnose(
|
|
1844
|
+
context,
|
|
1845
|
+
annotation,
|
|
1846
|
+
"CT_UNSUPPORTED_ANNOTATION",
|
|
1847
|
+
"Dynamic annotation effects are unknown; tracked state is invalidated.",
|
|
1848
|
+
)
|
|
1849
|
+
self._invalidate(list(context.env.values()), context, annotation)
|
|
1850
|
+
if node.decorator_list or context.scope != "<module>":
|
|
1851
|
+
bound = self._bound_names(node) | {
|
|
1852
|
+
arg.arg for arg in arguments + node.args.kwonlyargs
|
|
1853
|
+
}
|
|
1854
|
+
captures = [
|
|
1855
|
+
context.env[child.id]
|
|
1856
|
+
for child in ast.walk(node)
|
|
1857
|
+
if isinstance(child, ast.Name)
|
|
1858
|
+
and isinstance(child.ctx, ast.Load)
|
|
1859
|
+
and child.id not in bound
|
|
1860
|
+
and child.id in context.env
|
|
1861
|
+
]
|
|
1862
|
+
context.env[node.name] = _Value(escaped=self._escaped(captures))
|
|
1863
|
+
self._diagnose(
|
|
1864
|
+
context,
|
|
1865
|
+
node,
|
|
1866
|
+
"CT_UNSUPPORTED_FUNCTION",
|
|
1867
|
+
"Nested/decorated function body is unresolved; defining it does not "
|
|
1868
|
+
"execute its body. Defaults and decorators are analyzed separately.",
|
|
1869
|
+
)
|
|
1870
|
+
continue
|
|
1871
|
+
frame_parameters = {
|
|
1872
|
+
argument.arg
|
|
1873
|
+
for argument in arguments + node.args.kwonlyargs
|
|
1874
|
+
if self._dataframe_annotation(argument.annotation, context)
|
|
1875
|
+
}
|
|
1876
|
+
function = _Function(context.module, node, defaults, frame_parameters)
|
|
1877
|
+
self.functions.append(function)
|
|
1878
|
+
context.env[node.name] = _Value("function", function=function)
|
|
1879
|
+
elif isinstance(node, ast.Return):
|
|
1880
|
+
value = self._expr(node.value, context) if node.value else _Value()
|
|
1881
|
+
event = self._consume(value, context, node, "return")
|
|
1882
|
+
if event is not None and value.frame is not None:
|
|
1883
|
+
value.frame.history = (event,)
|
|
1884
|
+
value = replace(value, history=(event,))
|
|
1885
|
+
return value
|
|
1886
|
+
elif isinstance(node, (ast.Pass, ast.Assert)):
|
|
1887
|
+
if isinstance(node, ast.Assert):
|
|
1888
|
+
self._expr(node.test, context)
|
|
1889
|
+
self._diagnose(
|
|
1890
|
+
context,
|
|
1891
|
+
node,
|
|
1892
|
+
"CT_UNSUPPORTED_CONTROL_FLOW",
|
|
1893
|
+
"Assertions may terminate execution; reachability is not guaranteed.",
|
|
1894
|
+
)
|
|
1895
|
+
else:
|
|
1896
|
+
self._diagnose(
|
|
1897
|
+
context,
|
|
1898
|
+
node,
|
|
1899
|
+
"CT_UNSUPPORTED_CONTROL_FLOW",
|
|
1900
|
+
f"{type(node).__name__} is outside linear analysis; "
|
|
1901
|
+
"tracked state is invalidated.",
|
|
1902
|
+
)
|
|
1903
|
+
self._invalidate(list(context.env.values()), context, node)
|
|
1904
|
+
self._invalidate_api_writes(node, context)
|
|
1905
|
+
for name in self._bound_names(node):
|
|
1906
|
+
context.env[name] = _Value()
|
|
1907
|
+
return _Value()
|
|
1908
|
+
|
|
1909
|
+
def _invalidate_api_writes(self, node: ast.AST, context: _Context) -> None:
|
|
1910
|
+
for child in ast.walk(node):
|
|
1911
|
+
if isinstance(child, ast.Attribute) and isinstance(child.ctx, (ast.Store, ast.Del)):
|
|
1912
|
+
symbol = self._symbol(child, context)
|
|
1913
|
+
if symbol:
|
|
1914
|
+
self.api_mutations.add(symbol)
|
|
1915
|
+
|
|
1916
|
+
def _invoke(
|
|
1917
|
+
self,
|
|
1918
|
+
function: _Function,
|
|
1919
|
+
call: ast.Call,
|
|
1920
|
+
caller: _Context,
|
|
1921
|
+
positional: list[_Value],
|
|
1922
|
+
keywords: list[tuple[str | None, _Value]],
|
|
1923
|
+
*,
|
|
1924
|
+
template: bool = False,
|
|
1925
|
+
) -> _Value:
|
|
1926
|
+
node = function.node
|
|
1927
|
+
symbolic = template or caller.key.startswith("template:")
|
|
1928
|
+
identity = (function.module.name, node.name)
|
|
1929
|
+
invalid = bool(node.args.vararg or node.args.kwarg)
|
|
1930
|
+
invalid |= identity in caller.stack or len(caller.stack) >= self.MAX_DEPTH
|
|
1931
|
+
invalid |= self.call_count >= self.MAX_CALLS
|
|
1932
|
+
invalid |= any(isinstance(arg, ast.Starred) for arg in call.args)
|
|
1933
|
+
invalid |= any(name is None for name, _ in keywords)
|
|
1934
|
+
invalid |= any(
|
|
1935
|
+
isinstance(child, (ast.Global, ast.Nonlocal, ast.Yield, ast.YieldFrom))
|
|
1936
|
+
for child in ast.walk(node)
|
|
1937
|
+
)
|
|
1938
|
+
arguments = node.args.posonlyargs + node.args.args
|
|
1939
|
+
names = [arg.arg for arg in arguments]
|
|
1940
|
+
values = dict(function.defaults)
|
|
1941
|
+
if symbolic:
|
|
1942
|
+
values = {name: self._isolated(value) for name, value in values.items()}
|
|
1943
|
+
invalid |= len(positional) > len(arguments)
|
|
1944
|
+
assigned: set[str] = set()
|
|
1945
|
+
for name, value in zip(names, positional, strict=False):
|
|
1946
|
+
values[name] = value
|
|
1947
|
+
assigned.add(name)
|
|
1948
|
+
for keyword_name, value in keywords:
|
|
1949
|
+
if keyword_name is None:
|
|
1950
|
+
continue
|
|
1951
|
+
invalid |= keyword_name in assigned or keyword_name in {
|
|
1952
|
+
arg.arg for arg in node.args.posonlyargs
|
|
1953
|
+
}
|
|
1954
|
+
invalid |= keyword_name not in names + [arg.arg for arg in node.args.kwonlyargs]
|
|
1955
|
+
values[keyword_name] = value
|
|
1956
|
+
assigned.add(keyword_name)
|
|
1957
|
+
invalid |= any(arg.arg not in values for arg in arguments + node.args.kwonlyargs)
|
|
1958
|
+
# Early returns cannot be flattened into a single execution path.
|
|
1959
|
+
invalid |= any(isinstance(statement, ast.Return) for statement in node.body[:-1])
|
|
1960
|
+
if invalid:
|
|
1961
|
+
self._diagnose(
|
|
1962
|
+
caller,
|
|
1963
|
+
call,
|
|
1964
|
+
"CT_UNRESOLVED_CALL",
|
|
1965
|
+
"Call signature, recursion, expansion budget, or function body is unsupported.",
|
|
1966
|
+
)
|
|
1967
|
+
inputs = positional + list(values.values())
|
|
1968
|
+
if not symbolic:
|
|
1969
|
+
# An unsupported body can mutate defaults or module objects
|
|
1970
|
+
# without receiving them as explicit call arguments.
|
|
1971
|
+
inputs.extend(function.module.env.values())
|
|
1972
|
+
for value in inputs:
|
|
1973
|
+
self._consume(value, caller, call, "opaque_call")
|
|
1974
|
+
self._invalidate(inputs, caller, call)
|
|
1975
|
+
effects_env = dict(function.module.env)
|
|
1976
|
+
for name in self._bound_names(node):
|
|
1977
|
+
effects_env[name] = _Value()
|
|
1978
|
+
effects_env.update(values)
|
|
1979
|
+
effects_context = _Context(
|
|
1980
|
+
function.module, node.name, caller.key, effects_env, caller.stack
|
|
1981
|
+
)
|
|
1982
|
+
self._invalidate_api_writes(node, effects_context)
|
|
1983
|
+
if not symbolic:
|
|
1984
|
+
for child in ast.walk(node):
|
|
1985
|
+
if isinstance(child, ast.Global):
|
|
1986
|
+
for name in child.names:
|
|
1987
|
+
value = function.module.env.get(name, _Value())
|
|
1988
|
+
self._invalidate([value])
|
|
1989
|
+
function.module.env[name] = _Value()
|
|
1990
|
+
return _Value()
|
|
1991
|
+
function.called = True
|
|
1992
|
+
self.call_count += 1
|
|
1993
|
+
env = dict(function.module.env)
|
|
1994
|
+
if symbolic:
|
|
1995
|
+
env = {name: self._isolated(value) for name, value in env.items()}
|
|
1996
|
+
for name in self._bound_names(node):
|
|
1997
|
+
env[name] = _Value()
|
|
1998
|
+
env.update(values)
|
|
1999
|
+
location = self._location(caller, call)
|
|
2000
|
+
key = (
|
|
2001
|
+
f"template:{function.module.name}.{node.name}"
|
|
2002
|
+
if template
|
|
2003
|
+
else f"{caller.key}>{function.module.name}.{node.name}@"
|
|
2004
|
+
f"{location.path}:{location.line}:{location.column}"
|
|
2005
|
+
)
|
|
2006
|
+
supplied = list(values.values())
|
|
2007
|
+
call_event = self._event(
|
|
2008
|
+
caller,
|
|
2009
|
+
call,
|
|
2010
|
+
"call",
|
|
2011
|
+
(
|
|
2012
|
+
f"Analyze function {function.module.name}.{node.name}"
|
|
2013
|
+
if template
|
|
2014
|
+
else f"Call {function.module.name}.{node.name}"
|
|
2015
|
+
),
|
|
2016
|
+
frames=self._escaped(supplied),
|
|
2017
|
+
inputs=tuple(dict.fromkeys(item for value in supplied for item in value.dependencies)),
|
|
2018
|
+
parents=tuple(
|
|
2019
|
+
dict.fromkeys(
|
|
2020
|
+
caller.parent_history
|
|
2021
|
+
+ tuple(item for value in supplied for item in self._parents(value))
|
|
2022
|
+
)
|
|
2023
|
+
),
|
|
2024
|
+
expression=(
|
|
2025
|
+
"Symbolic function analysis; no resolved caller"
|
|
2026
|
+
if template
|
|
2027
|
+
else self._code(caller, call)
|
|
2028
|
+
)
|
|
2029
|
+
+ "\nParameters: "
|
|
2030
|
+
+ ", ".join(f"{name} <- {value.expression or name}" for name, value in values.items()),
|
|
2031
|
+
evidence="inferred" if symbolic else "confirmed",
|
|
2032
|
+
)
|
|
2033
|
+
context = _Context(
|
|
2034
|
+
function.module, node.name, key, env, caller.stack + (identity,), (call_event,)
|
|
2035
|
+
)
|
|
2036
|
+
for value in supplied:
|
|
2037
|
+
if value.frame is not None:
|
|
2038
|
+
value.frame.history = tuple(dict.fromkeys(value.frame.history + (call_event,)))
|
|
2039
|
+
for argument in (*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs):
|
|
2040
|
+
value = values[argument.arg]
|
|
2041
|
+
if value.frame is not None:
|
|
2042
|
+
# Preserve the proven parameter binding in the callee context;
|
|
2043
|
+
# identical parameter names in unrelated calls stay separate.
|
|
2044
|
+
target = ast.copy_location(ast.Name(id=argument.arg, ctx=ast.Store()), argument)
|
|
2045
|
+
self._bind(target, value, context)
|
|
2046
|
+
for value in positional + [value for _, value in keywords]:
|
|
2047
|
+
self._consume(value, caller, call, "argument")
|
|
2048
|
+
result = self._statements(node.body, context)
|
|
2049
|
+
if result.frame is not None:
|
|
2050
|
+
return result
|
|
2051
|
+
return _Value(escaped=self._escaped([result]))
|
|
2052
|
+
|
|
2053
|
+
def _module(self, module: _Module) -> None:
|
|
2054
|
+
if module.status != "new":
|
|
2055
|
+
return
|
|
2056
|
+
module.status = "visiting"
|
|
2057
|
+
context = _Context(module, "<module>", f"module:{module.name}", module.env)
|
|
2058
|
+
try:
|
|
2059
|
+
self._statements(module.tree.body, context)
|
|
2060
|
+
except RecursionError:
|
|
2061
|
+
self._diagnose(
|
|
2062
|
+
context, module.tree, "CT_ANALYSIS_LIMIT", "AST interpretation depth was exceeded."
|
|
2063
|
+
)
|
|
2064
|
+
self._invalidate(list(context.env.values()))
|
|
2065
|
+
module.status = "done"
|
|
2066
|
+
|
|
2067
|
+
def run(self) -> None:
|
|
2068
|
+
for module in self.modules.values():
|
|
2069
|
+
self._module(module)
|
|
2070
|
+
for function in list(self.functions):
|
|
2071
|
+
if function.called:
|
|
2072
|
+
continue
|
|
2073
|
+
module = function.module
|
|
2074
|
+
node = function.node
|
|
2075
|
+
context = _Context(module, node.name, f"template:{module.name}.{node.name}", module.env)
|
|
2076
|
+
values: list[_Value] = []
|
|
2077
|
+
keywords: list[tuple[str | None, _Value]] = []
|
|
2078
|
+
for argument in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
|
|
2079
|
+
if argument.arg in function.frame_parameters:
|
|
2080
|
+
state = self._new_frame(
|
|
2081
|
+
context, argument, f"parameter:{argument.arg}", "inferred"
|
|
2082
|
+
)
|
|
2083
|
+
value = self._frame_value(state)
|
|
2084
|
+
else:
|
|
2085
|
+
value = self._isolated(function.defaults.get(argument.arg, _Value()))
|
|
2086
|
+
if argument in node.args.kwonlyargs:
|
|
2087
|
+
keywords.append((argument.arg, value))
|
|
2088
|
+
else:
|
|
2089
|
+
values.append(value)
|
|
2090
|
+
call = ast.Call(func=ast.Name(id=node.name), args=[], keywords=[])
|
|
2091
|
+
ast.copy_location(call, node)
|
|
2092
|
+
mutations = set(self.api_mutations)
|
|
2093
|
+
try:
|
|
2094
|
+
self._invoke(function, call, context, values, keywords, template=True)
|
|
2095
|
+
except RecursionError:
|
|
2096
|
+
self._diagnose(
|
|
2097
|
+
context, node, "CT_ANALYSIS_LIMIT", "Function AST traversal depth was exceeded."
|
|
2098
|
+
)
|
|
2099
|
+
self._invalidate(values + [value for _, value in keywords])
|
|
2100
|
+
finally:
|
|
2101
|
+
self.api_mutations = mutations
|
|
2102
|
+
|
|
2103
|
+
@staticmethod
|
|
2104
|
+
def _isolated(value: _Value) -> _Value:
|
|
2105
|
+
if (
|
|
2106
|
+
value.frame is not None
|
|
2107
|
+
or value.dependencies
|
|
2108
|
+
or value.escaped
|
|
2109
|
+
or value.epochs
|
|
2110
|
+
or value.components is not None
|
|
2111
|
+
or value.index_key is not None
|
|
2112
|
+
):
|
|
2113
|
+
return _Value()
|
|
2114
|
+
return value
|
|
2115
|
+
|
|
2116
|
+
def _dataframe_annotation(self, annotation: ast.AST | None, context: _Context) -> bool:
|
|
2117
|
+
if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str):
|
|
2118
|
+
try:
|
|
2119
|
+
annotation = ast.parse(annotation.value, mode="eval").body
|
|
2120
|
+
except (SyntaxError, RecursionError):
|
|
2121
|
+
return False
|
|
2122
|
+
return annotation is not None and self._symbol(annotation, context) == "pandas.DataFrame"
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def analyze_project(path: str | Path, *, config: ScanConfig | None = None) -> LineageReport:
|
|
2126
|
+
"""Build static lineage for resolved, linear Pandas code without executing it.
|
|
2127
|
+
|
|
2128
|
+
``scan_project`` remains available for the broader occurrence inventory. This
|
|
2129
|
+
report contains graph facts only where the lineage interpreter has evidence.
|
|
2130
|
+
"""
|
|
2131
|
+
root = Path(path).resolve()
|
|
2132
|
+
discovery = scan_project(root, config=config)
|
|
2133
|
+
modules: dict[str, _Module] = {}
|
|
2134
|
+
ambiguous: set[str] = set()
|
|
2135
|
+
diagnostics = [
|
|
2136
|
+
diagnostic
|
|
2137
|
+
for diagnostic in discovery.diagnostics
|
|
2138
|
+
if diagnostic.code
|
|
2139
|
+
in {
|
|
2140
|
+
"CT_DISCOVERY_ERROR",
|
|
2141
|
+
"CT_READ_ERROR",
|
|
2142
|
+
"CT_ENCODING_ERROR",
|
|
2143
|
+
"CT_SYNTAX_ERROR",
|
|
2144
|
+
"CT_ANALYSIS_LIMIT",
|
|
2145
|
+
}
|
|
2146
|
+
]
|
|
2147
|
+
limited_files = {item.location.path for item in diagnostics if item.code == "CT_ANALYSIS_LIMIT"}
|
|
2148
|
+
for file in discovery.files:
|
|
2149
|
+
if file.status != "parsed" or file.path in limited_files:
|
|
2150
|
+
continue
|
|
2151
|
+
try:
|
|
2152
|
+
raw = (root / file.path).read_bytes()
|
|
2153
|
+
if hashlib.sha256(raw).hexdigest() != file.sha256:
|
|
2154
|
+
raise ValueError("Source changed during analysis; rescan a stable project.")
|
|
2155
|
+
encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline)
|
|
2156
|
+
source = raw.decode(encoding).replace("\r\n", "\n").replace("\r", "\n")
|
|
2157
|
+
tree = ast.parse(source, filename=file.path)
|
|
2158
|
+
except (OSError, UnicodeError, LookupError, SyntaxError, RecursionError, ValueError) as exc:
|
|
2159
|
+
diagnostics.append(
|
|
2160
|
+
Diagnostic("CT_READ_ERROR", "error", str(exc), Location(file.path, 1, 0))
|
|
2161
|
+
)
|
|
2162
|
+
continue
|
|
2163
|
+
if file.module in modules or file.module in ambiguous:
|
|
2164
|
+
diagnostics.append(
|
|
2165
|
+
Diagnostic(
|
|
2166
|
+
"CT_AMBIGUOUS_MODULE",
|
|
2167
|
+
"error",
|
|
2168
|
+
"Multiple files resolve to this module name.",
|
|
2169
|
+
Location(file.path, 1, 0),
|
|
2170
|
+
)
|
|
2171
|
+
)
|
|
2172
|
+
ambiguous.add(file.module)
|
|
2173
|
+
modules.pop(file.module, None)
|
|
2174
|
+
continue
|
|
2175
|
+
modules[file.module] = _Module(file.module, file.path, source, tree)
|
|
2176
|
+
modules = {
|
|
2177
|
+
name: module
|
|
2178
|
+
for name, module in modules.items()
|
|
2179
|
+
if not any(name.startswith(prefix + ".") for prefix in ambiguous)
|
|
2180
|
+
}
|
|
2181
|
+
analyzer = _Analyzer(modules, ambiguous)
|
|
2182
|
+
analyzer.run()
|
|
2183
|
+
diagnostics.extend(analyzer.diagnostics)
|
|
2184
|
+
return LineageReport(
|
|
2185
|
+
files=discovery.files,
|
|
2186
|
+
functions=discovery.functions,
|
|
2187
|
+
dataframes=tuple(analyzer.frames),
|
|
2188
|
+
columns=tuple(analyzer.columns.values()),
|
|
2189
|
+
operations=tuple(analyzer.operations),
|
|
2190
|
+
edges=tuple(analyzer.edges),
|
|
2191
|
+
consumptions=tuple(analyzer.consumptions),
|
|
2192
|
+
history=tuple(analyzer.history),
|
|
2193
|
+
diagnostics=tuple(
|
|
2194
|
+
sorted(
|
|
2195
|
+
diagnostics,
|
|
2196
|
+
key=lambda diagnostic: (
|
|
2197
|
+
diagnostic.location.path,
|
|
2198
|
+
diagnostic.location.line,
|
|
2199
|
+
diagnostic.location.column,
|
|
2200
|
+
diagnostic.code,
|
|
2201
|
+
diagnostic.message,
|
|
2202
|
+
),
|
|
2203
|
+
)
|
|
2204
|
+
),
|
|
2205
|
+
exclude=discovery.exclude,
|
|
2206
|
+
)
|