coltrace 0.1.0.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
coltrace/analysis.py ADDED
@@ -0,0 +1,2206 @@
1
+ """Conservative, ordered interpretation of a deliberately small Pandas subset.
2
+
3
+ Only ASTs and abstract values are interpreted. Project code and data sources are
4
+ never imported, evaluated, or opened by this module.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import ast
10
+ import hashlib
11
+ import io
12
+ import tokenize
13
+ from dataclasses import dataclass, field, replace
14
+ from pathlib import Path
15
+ from typing import Literal
16
+
17
+ from .callbacks import summarize_callback
18
+ from .lineage_models import (
19
+ Column,
20
+ Completeness,
21
+ Confidence,
22
+ Consumption,
23
+ Edge,
24
+ Frame,
25
+ HistoryEvent,
26
+ LineageReport,
27
+ Operation,
28
+ )
29
+ from .models import Diagnostic, Location
30
+ from .scanner import ScanConfig, scan_project
31
+
32
+
33
+ def _identifier(prefix: str, *parts: object) -> str:
34
+ digest = hashlib.sha256(repr(parts).encode("utf-8")).hexdigest()[:24]
35
+ return f"{prefix}:{digest}"
36
+
37
+
38
+ def _confidence(*values: Confidence) -> Confidence:
39
+ if "unresolved" in values:
40
+ return "unresolved"
41
+ return "inferred" if "inferred" in values else "confirmed"
42
+
43
+
44
+ def _completeness(*values: Completeness) -> Completeness:
45
+ if "unresolved" in values:
46
+ return "partial" if "complete" in values or "partial" in values else "unresolved"
47
+ return "partial" if "partial" in values else "complete"
48
+
49
+
50
+ @dataclass
51
+ class _Value:
52
+ kind: Literal[
53
+ "unknown",
54
+ "literal",
55
+ "series",
56
+ "frame",
57
+ "symbol",
58
+ "function",
59
+ "rows",
60
+ "index",
61
+ "mask",
62
+ "accessor",
63
+ ] = "unknown"
64
+ frame: _State | None = None
65
+ dependencies: tuple[str, ...] = ()
66
+ symbol: str = ""
67
+ function: _Function | None = None
68
+ evidence: Confidence = "confirmed"
69
+ completeness: Completeness = "unresolved"
70
+ escaped: tuple[str, ...] = ()
71
+ epochs: tuple[tuple[str, int], ...] = ()
72
+ components: tuple[_Value, ...] | None = None
73
+ index_key: str | None = None
74
+ producer: str = "derive"
75
+ expression: str = ""
76
+ history: tuple[str, ...] = ()
77
+
78
+
79
+ @dataclass
80
+ class _Module:
81
+ name: str
82
+ path: str
83
+ source: str
84
+ tree: ast.Module
85
+ env: dict[str, _Value] = field(default_factory=dict)
86
+ status: Literal["new", "visiting", "done"] = "new"
87
+
88
+
89
+ @dataclass
90
+ class _Function:
91
+ module: _Module
92
+ node: ast.FunctionDef
93
+ defaults: dict[str, _Value]
94
+ frame_parameters: set[str]
95
+ called: bool = False
96
+
97
+
98
+ @dataclass
99
+ class _Context:
100
+ module: _Module
101
+ scope: str
102
+ key: str
103
+ env: dict[str, _Value]
104
+ stack: tuple[tuple[str, str], ...] = ()
105
+ parent_history: tuple[str, ...] = ()
106
+
107
+
108
+ @dataclass
109
+ class _Snapshot:
110
+ state: _State
111
+ current: dict[str, str]
112
+ fallback: _Copy | None
113
+ schema: set[str] | None
114
+ epoch: int
115
+ tainted: bool
116
+ versions: dict[str, int]
117
+ removed: set[str]
118
+ history: tuple[str, ...]
119
+
120
+
121
+ @dataclass
122
+ class _Copy:
123
+ source: _Snapshot
124
+ names: dict[str, str]
125
+ context: _Context
126
+ node: ast.AST
127
+ evidence: Confidence
128
+
129
+
130
+ @dataclass
131
+ class _State:
132
+ record: Frame
133
+ current: dict[str, str] = field(default_factory=dict)
134
+ versions: dict[str, int] = field(default_factory=dict)
135
+ fallback: _Copy | None = None
136
+ schema: set[str] | None = None
137
+ epoch: int = 0
138
+ tainted: bool = False
139
+ removed: set[str] = field(default_factory=set)
140
+ index_key: str = ""
141
+ positional: tuple[_Value, ...] | None = None
142
+ history: tuple[str, ...] = ()
143
+
144
+
145
+ class _Analyzer:
146
+ MAX_CALLS = 1000
147
+ MAX_DEPTH = 32
148
+
149
+ def __init__(self, modules: dict[str, _Module], ambiguous: set[str]) -> None:
150
+ self.modules = modules
151
+ self.ambiguous = ambiguous
152
+ self.frames: list[Frame] = []
153
+ self.states: dict[str, _State] = {}
154
+ self.columns: dict[str, Column] = {}
155
+ self.operations: list[Operation] = []
156
+ self.edges: list[Edge] = []
157
+ self.consumptions: list[Consumption] = []
158
+ self.diagnostics: list[Diagnostic] = []
159
+ self.functions: list[_Function] = []
160
+ self.call_count = 0
161
+ self.api_mutations: set[str] = set()
162
+ self.history: list[HistoryEvent] = []
163
+ self.column_history: dict[str, str] = {}
164
+ self.frame_boundaries: dict[str, str] = {}
165
+
166
+ @staticmethod
167
+ def _code(context: _Context, node: ast.AST) -> str:
168
+ return ast.get_source_segment(context.module.source, node) or ast.unparse(node)
169
+
170
+ def _event(
171
+ self,
172
+ context: _Context,
173
+ node: ast.AST,
174
+ kind: str,
175
+ label: str,
176
+ *,
177
+ frames: tuple[str, ...] = (),
178
+ inputs: tuple[str, ...] = (),
179
+ outputs: tuple[str, ...] = (),
180
+ parents: tuple[str, ...] = (),
181
+ expression: str | None = None,
182
+ evidence: Confidence = "confirmed",
183
+ completeness: Completeness = "complete",
184
+ ) -> str:
185
+ sequence = len(self.history)
186
+ evidence = _confidence(
187
+ evidence, "inferred" if context.key.startswith("template:") else "confirmed"
188
+ )
189
+ parents = tuple(sorted(set(parents)))
190
+ event = HistoryEvent(
191
+ _identifier("history", context.key, self._location(context, node), kind, sequence),
192
+ sequence,
193
+ kind,
194
+ label,
195
+ self._location(context, node),
196
+ context.module.name,
197
+ context.scope,
198
+ context.key,
199
+ tuple(sorted(set(frames))),
200
+ tuple(sorted(set(inputs))),
201
+ tuple(sorted(set(outputs))),
202
+ self._code(context, node) if expression is None else expression,
203
+ evidence,
204
+ completeness,
205
+ parents,
206
+ )
207
+ self.history.append(event)
208
+ return event.id
209
+
210
+ def _parents(self, value: _Value) -> tuple[str, ...]:
211
+ return tuple(
212
+ dict.fromkeys(
213
+ value.history
214
+ + tuple(
215
+ self.column_history[item]
216
+ for item in value.dependencies
217
+ if item in self.column_history
218
+ )
219
+ )
220
+ )
221
+
222
+ @staticmethod
223
+ def _location(context: _Context, node: ast.AST) -> Location:
224
+ lines = context.module.source.split("\n")
225
+
226
+ def offset(line: int, byte: int) -> int:
227
+ if not 1 <= line <= len(lines):
228
+ return byte
229
+ return len(lines[line - 1].encode("utf-8")[:byte].decode("utf-8", errors="ignore"))
230
+
231
+ line = getattr(node, "lineno", 1)
232
+ end_line = getattr(node, "end_lineno", None)
233
+ end_col = getattr(node, "end_col_offset", None)
234
+ return Location(
235
+ context.module.path,
236
+ line,
237
+ offset(line, getattr(node, "col_offset", 0)),
238
+ end_line,
239
+ offset(end_line, end_col) if end_line is not None and end_col is not None else None,
240
+ )
241
+
242
+ def _diagnose(self, context: _Context, node: ast.AST, code: str, message: str) -> None:
243
+ self.diagnostics.append(
244
+ Diagnostic(
245
+ code,
246
+ "warning",
247
+ message,
248
+ self._location(context, node),
249
+ context.scope,
250
+ "Inspect this boundary; lineage beyond it may be incomplete.",
251
+ )
252
+ )
253
+
254
+ def _new_frame(
255
+ self,
256
+ context: _Context,
257
+ node: ast.AST,
258
+ origin: str,
259
+ evidence: Confidence = "confirmed",
260
+ ) -> _State:
261
+ location = self._location(context, node)
262
+ if context.key.startswith("template:"):
263
+ evidence = _confidence(evidence, "inferred")
264
+ record = Frame(
265
+ _identifier("df", location, context.key, origin),
266
+ origin,
267
+ location,
268
+ context.module.name,
269
+ context.scope,
270
+ context.key,
271
+ evidence,
272
+ )
273
+ state = _State(record, index_key=record.id)
274
+ self.frames.append(record)
275
+ self.states[record.id] = state
276
+ if origin.startswith("parameter:"):
277
+ state.history = (
278
+ self._event(
279
+ context,
280
+ node,
281
+ "parameter",
282
+ f"DataFrame parameter {origin.split(':', 1)[1]}",
283
+ frames=(record.id,),
284
+ evidence=evidence,
285
+ completeness="partial",
286
+ ),
287
+ )
288
+ return state
289
+
290
+ @staticmethod
291
+ def _frame_value(state: _State) -> _Value:
292
+ return _Value(
293
+ "frame",
294
+ frame=state,
295
+ evidence=state.record.evidence,
296
+ completeness="complete",
297
+ history=state.history,
298
+ )
299
+
300
+ def _operation(
301
+ self,
302
+ context: _Context,
303
+ node: ast.AST,
304
+ kind: str,
305
+ suffix: str,
306
+ evidence: Confidence,
307
+ completeness: Completeness,
308
+ ) -> Operation:
309
+ location = self._location(context, node)
310
+ operation = Operation(
311
+ _identifier("op", location, context.key, kind, suffix),
312
+ kind,
313
+ location,
314
+ context.module.name,
315
+ context.scope,
316
+ context.key,
317
+ evidence,
318
+ completeness,
319
+ )
320
+ self.operations.append(operation)
321
+ return operation
322
+
323
+ def _write(
324
+ self,
325
+ state: _State,
326
+ name: str,
327
+ value: _Value,
328
+ context: _Context,
329
+ node: ast.AST,
330
+ kind: str = "derive",
331
+ ) -> str:
332
+ value = self._refresh(value)
333
+ previous = state.current.get(name)
334
+ state.positional = None
335
+ if kind == "derive":
336
+ kind = value.producer
337
+ # Reserve the epoch's external version even when that column has not
338
+ # been observed yet: an older lazy copy can materialize it later.
339
+ version = state.versions.get(name, state.epoch) + 1
340
+ state.versions[name] = version
341
+ evidence = _confidence(state.record.evidence, value.evidence)
342
+ if value.kind not in {"literal", "series"}:
343
+ evidence = "unresolved"
344
+ kind = "opaque"
345
+ self._diagnose(
346
+ context,
347
+ node,
348
+ "CT_UNRESOLVED_VALUE",
349
+ "The assigned value has unresolved provenance; no hidden dependencies are assumed.",
350
+ )
351
+ column = Column(
352
+ _identifier("col", state.record.id, name, version),
353
+ state.record.id,
354
+ name,
355
+ version,
356
+ self._location(context, node),
357
+ context.module.name,
358
+ context.scope,
359
+ context.key,
360
+ "write" if kind != "source" else "source",
361
+ evidence,
362
+ value.completeness,
363
+ )
364
+ self.columns[column.id] = column
365
+ state.current[name] = column.id
366
+ state.removed.discard(name)
367
+ if state.schema is not None:
368
+ state.schema.add(name)
369
+ operation = self._operation(context, node, kind, column.id, evidence, value.completeness)
370
+ for dependency in dict.fromkeys(value.dependencies):
371
+ self.edges.append(Edge(dependency, operation.id, "read", evidence))
372
+ self.edges.append(Edge(operation.id, column.id, "write", evidence))
373
+ event_kind = "boundary" if kind == "opaque" else kind
374
+ label = f"{name} = {value.expression or kind}"
375
+ if previous is not None and kind not in {"rename", "source"}:
376
+ event_kind = "overwrite"
377
+ label = f"Overwrite {name}: {value.expression or kind}"
378
+ self.column_history[column.id] = self._event(
379
+ context,
380
+ node,
381
+ event_kind,
382
+ label,
383
+ frames=(state.record.id,),
384
+ inputs=value.dependencies,
385
+ outputs=(column.id,),
386
+ parents=tuple(dict.fromkeys(state.history + self._parents(value))),
387
+ expression=f"{self._code(context, node)} = {value.expression}"
388
+ if value.expression
389
+ else None,
390
+ evidence=evidence,
391
+ completeness=value.completeness,
392
+ )
393
+ return column.id
394
+
395
+ def _external(self, snapshot: _Snapshot, name: str, context: _Context, node: ast.AST) -> str:
396
+ state = snapshot.state
397
+ identifier = _identifier("external", state.record.id, name, snapshot.epoch)
398
+ if identifier not in self.columns:
399
+ evidence = "unresolved" if snapshot.tainted else state.record.evidence
400
+ version = snapshot.versions.get(name, snapshot.epoch) if snapshot.tainted else 0
401
+ state.versions[name] = max(state.versions.get(name, -1), version)
402
+ self.columns[identifier] = Column(
403
+ identifier,
404
+ state.record.id,
405
+ name,
406
+ version,
407
+ self._location(context, node),
408
+ context.module.name,
409
+ context.scope,
410
+ context.key,
411
+ "opaque" if snapshot.tainted else "external_unknown",
412
+ evidence,
413
+ "unresolved" if snapshot.tainted else "partial",
414
+ )
415
+ self.column_history[identifier] = self._event(
416
+ context,
417
+ node,
418
+ "boundary" if snapshot.tainted else "external",
419
+ (
420
+ f"Provenance lost before observing {name}"
421
+ if snapshot.tainted
422
+ else f"Observe input column {name}; definition in external data is unknown"
423
+ ),
424
+ frames=(state.record.id,),
425
+ outputs=(identifier,),
426
+ parents=snapshot.history,
427
+ evidence=evidence,
428
+ completeness="unresolved" if snapshot.tainted else "partial",
429
+ )
430
+ snapshot.current[name] = identifier
431
+ return identifier
432
+
433
+ @staticmethod
434
+ def _snapshot(state: _State) -> _Snapshot:
435
+ return _Snapshot(
436
+ state,
437
+ dict(state.current),
438
+ state.fallback,
439
+ None if state.schema is None else set(state.schema),
440
+ state.epoch,
441
+ state.tainted,
442
+ dict(state.versions),
443
+ set(state.removed),
444
+ state.history,
445
+ )
446
+
447
+ def _lookup_snapshot(
448
+ self, snapshot: _Snapshot, name: str, context: _Context, node: ast.AST
449
+ ) -> str | None:
450
+ if name in snapshot.current:
451
+ return snapshot.current[name]
452
+ if name in snapshot.removed or (
453
+ snapshot.schema is not None and name not in snapshot.schema
454
+ ):
455
+ self._diagnose(
456
+ context,
457
+ node,
458
+ "CT_MISSING_COLUMN",
459
+ f"Column {name!r} is absent from the known schema.",
460
+ )
461
+ return None
462
+ if snapshot.fallback is not None:
463
+ copy = snapshot.fallback
464
+ source_name = copy.names.get(name, name)
465
+ source = self._lookup_snapshot(copy.source, source_name, copy.context, copy.node)
466
+ if source is None:
467
+ return None
468
+ evidence = _confidence(copy.evidence, self.columns[source].evidence)
469
+ identifier = _identifier("copy", snapshot.state.record.id, name, snapshot.epoch)
470
+ if identifier not in self.columns:
471
+ record = snapshot.state.record
472
+ version = 0
473
+ snapshot.state.versions[name] = max(snapshot.state.versions.get(name, -1), version)
474
+ column = Column(
475
+ identifier,
476
+ record.id,
477
+ name,
478
+ version,
479
+ self._location(copy.context, copy.node),
480
+ copy.context.module.name,
481
+ copy.context.scope,
482
+ copy.context.key,
483
+ "copy",
484
+ evidence,
485
+ self.columns[source].completeness,
486
+ )
487
+ self.columns[identifier] = column
488
+ operation = self._operation(
489
+ copy.context,
490
+ copy.node,
491
+ "rename" if name in copy.names else "pass",
492
+ identifier,
493
+ evidence,
494
+ column.completeness,
495
+ )
496
+ self.edges.extend(
497
+ [
498
+ Edge(source, operation.id, "read", evidence),
499
+ Edge(operation.id, identifier, "write", evidence),
500
+ ]
501
+ )
502
+ kind = "rename" if name in copy.names else record.origin
503
+ self.column_history[identifier] = self._event(
504
+ copy.context,
505
+ copy.node,
506
+ kind,
507
+ f"{source_name} -> {name}"
508
+ if kind == "rename"
509
+ else f"Preserve {name} through {kind}",
510
+ frames=(copy.source.state.record.id, record.id),
511
+ inputs=(source,),
512
+ outputs=(identifier,),
513
+ parents=tuple(
514
+ dict.fromkeys(
515
+ snapshot.history
516
+ + (
517
+ (self.column_history[source],)
518
+ if source in self.column_history
519
+ else ()
520
+ )
521
+ )
522
+ ),
523
+ evidence=evidence,
524
+ completeness=column.completeness,
525
+ )
526
+ snapshot.current[name] = identifier
527
+ return identifier
528
+ return self._external(snapshot, name, context, node)
529
+
530
+ def _read(
531
+ self,
532
+ state: _State,
533
+ name: str,
534
+ context: _Context,
535
+ node: ast.AST,
536
+ *,
537
+ record: bool = True,
538
+ ) -> _Value:
539
+ if state.positional is not None and state.schema is None:
540
+ self._diagnose(
541
+ context,
542
+ node,
543
+ "CT_UNSUPPORTED_POSITIONAL_COLUMN",
544
+ "Unlabeled row-result columns require a matching multi-column assignment.",
545
+ )
546
+ return _Value()
547
+ snapshot = self._snapshot(state)
548
+ identifier = self._lookup_snapshot(snapshot, name, context, node)
549
+ if identifier is None:
550
+ return _Value()
551
+ state.current[name] = identifier
552
+ self.consumptions.append(
553
+ Consumption(
554
+ identifier,
555
+ self._location(context, node),
556
+ context.module.name,
557
+ context.scope,
558
+ context.key,
559
+ )
560
+ )
561
+ column = self.columns[identifier]
562
+ parents = (self.column_history[identifier],) if identifier in self.column_history else ()
563
+ if record:
564
+ parents = (
565
+ self._event(
566
+ context,
567
+ node,
568
+ "read",
569
+ f"Read column {name}",
570
+ frames=(state.record.id,),
571
+ inputs=(identifier,),
572
+ parents=parents,
573
+ evidence=column.evidence,
574
+ completeness=column.completeness,
575
+ ),
576
+ )
577
+ return _Value(
578
+ "series",
579
+ dependencies=(identifier,),
580
+ evidence=column.evidence,
581
+ completeness=column.completeness,
582
+ epochs=((state.record.id, state.epoch),),
583
+ index_key=state.index_key,
584
+ history=parents,
585
+ )
586
+
587
+ def _refresh(self, value: _Value) -> _Value:
588
+ if any(self.states[identifier].epoch != epoch for identifier, epoch in value.epochs):
589
+ return _Value(
590
+ value.kind,
591
+ evidence="unresolved",
592
+ completeness="unresolved",
593
+ escaped=tuple(
594
+ dict.fromkeys(
595
+ value.escaped + tuple(identifier for identifier, _ in value.epochs)
596
+ )
597
+ ),
598
+ )
599
+ return value
600
+
601
+ def _invalidate(
602
+ self,
603
+ values: list[_Value],
604
+ context: _Context | None = None,
605
+ node: ast.AST | None = None,
606
+ ) -> None:
607
+ states: dict[str, _State] = {}
608
+ for value in values:
609
+ if value.frame is not None:
610
+ states[value.frame.record.id] = value.frame
611
+ for dependency in value.dependencies:
612
+ frame_id = self.columns[dependency].dataframe_id
613
+ states[frame_id] = self.states[frame_id]
614
+ for frame_id in value.escaped:
615
+ states[frame_id] = self.states[frame_id]
616
+ if states and context is not None and node is not None:
617
+ location = self._location(context, node)
618
+ prior = {event.id: event for event in self.history}
619
+ already = all(
620
+ state.record.id in self.frame_boundaries
621
+ and prior[self.frame_boundaries[state.record.id]].location == location
622
+ and prior[self.frame_boundaries[state.record.id]].context == context.key
623
+ for state in states.values()
624
+ )
625
+ if not already:
626
+ diagnostic = next(
627
+ (
628
+ item
629
+ for item in reversed(self.diagnostics)
630
+ if item.location == location and item.scope == context.scope
631
+ ),
632
+ None,
633
+ )
634
+ event = self._event(
635
+ context,
636
+ node,
637
+ "boundary",
638
+ f"Provenance interrupted: {diagnostic.code}"
639
+ if diagnostic
640
+ else "Unresolved mutation effects",
641
+ frames=tuple(states),
642
+ # Forgetting abstract state is an analysis boundary, not
643
+ # evidence that the statement read every known column.
644
+ parents=tuple(
645
+ dict.fromkeys(
646
+ parent for state in states.values() for parent in state.history
647
+ )
648
+ ),
649
+ evidence="unresolved",
650
+ completeness="unresolved",
651
+ )
652
+ for frame_id in states:
653
+ self.frame_boundaries[frame_id] = event
654
+ for state in states.values():
655
+ state.current.clear()
656
+ state.fallback = None
657
+ state.schema = None
658
+ state.removed.clear()
659
+ state.epoch += 1
660
+ state.index_key = _identifier("index", state.record.id, state.epoch)
661
+ state.positional = None
662
+ state.tainted = True
663
+ if state.record.id in self.frame_boundaries:
664
+ state.history = (self.frame_boundaries[state.record.id],)
665
+ state.versions = {name: version + 1 for name, version in state.versions.items()}
666
+
667
+ def _escaped(self, values: list[_Value]) -> tuple[str, ...]:
668
+ identifiers: list[str] = []
669
+ for value in values:
670
+ identifiers.extend(value.escaped)
671
+ if value.frame is not None:
672
+ identifiers.append(value.frame.record.id)
673
+ identifiers.extend(self.columns[column].dataframe_id for column in value.dependencies)
674
+ return tuple(dict.fromkeys(identifiers))
675
+
676
+ def _consume(self, value: _Value, context: _Context, node: ast.AST, kind: str) -> str | None:
677
+ dependencies = value.dependencies
678
+ if value.frame is not None:
679
+ dependencies = tuple(value.frame.current.values())
680
+ for identifier in value.escaped:
681
+ dependencies += tuple(self.states[identifier].current.values())
682
+ for identifier in dict.fromkeys(dependencies):
683
+ self.consumptions.append(
684
+ Consumption(
685
+ identifier,
686
+ self._location(context, node),
687
+ context.module.name,
688
+ context.scope,
689
+ context.key,
690
+ kind,
691
+ )
692
+ )
693
+ frames = self._escaped([value])
694
+ if dependencies or frames:
695
+ event_kind = (
696
+ "boundary"
697
+ if kind == "opaque_call"
698
+ else (kind if kind in {"sink", "return", "filter"} else "consume")
699
+ )
700
+ event = self._event(
701
+ context,
702
+ node,
703
+ event_kind,
704
+ kind.replace("_", " "),
705
+ frames=frames,
706
+ inputs=dependencies,
707
+ parents=self._parents(value)
708
+ + (value.frame.history if value.frame is not None else ()),
709
+ evidence="unresolved" if kind == "opaque_call" else value.evidence,
710
+ completeness="unresolved" if kind == "opaque_call" else value.completeness,
711
+ expression=f"Callback {node.name}(...)"
712
+ if isinstance(node, ast.FunctionDef)
713
+ else None,
714
+ )
715
+ if event_kind == "boundary":
716
+ for frame_id in frames:
717
+ self.frame_boundaries[frame_id] = event
718
+ return event
719
+ return None
720
+
721
+ def _symbol(self, node: ast.AST, context: _Context) -> str:
722
+ if isinstance(node, ast.Name):
723
+ value = context.env.get(node.id, _Value())
724
+ return value.symbol if value.kind == "symbol" else ""
725
+ if isinstance(node, ast.Attribute):
726
+ base = self._symbol(node.value, context)
727
+ symbol = f"{base}.{node.attr}" if base else ""
728
+ if any(
729
+ symbol == changed or symbol.startswith(changed + ".")
730
+ for changed in self.api_mutations
731
+ ):
732
+ return ""
733
+ return symbol
734
+ return ""
735
+
736
+ @staticmethod
737
+ def _literal(node: ast.AST) -> bool:
738
+ if isinstance(node, ast.Constant):
739
+ return True
740
+ if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
741
+ return all(_Analyzer._literal(item) for item in node.elts)
742
+ if isinstance(node, ast.Dict):
743
+ return all(key is not None and _Analyzer._literal(key) for key in node.keys) and all(
744
+ _Analyzer._literal(value) for value in node.values
745
+ )
746
+ return (
747
+ isinstance(node, ast.UnaryOp)
748
+ and isinstance(node.op, (ast.UAdd, ast.USub))
749
+ and (_Analyzer._literal(node.operand))
750
+ )
751
+
752
+ @staticmethod
753
+ def _column_target(node: ast.Subscript) -> tuple[ast.expr, str | None]:
754
+ receiver = node.value
755
+ label: ast.AST = node.slice
756
+ if isinstance(receiver, ast.Attribute) and receiver.attr == "loc":
757
+ receiver = receiver.value
758
+ if not isinstance(label, ast.Tuple) or len(label.elts) != 2:
759
+ return receiver, None
760
+ rows, label = label.elts
761
+ if not isinstance(rows, ast.Slice) or any((rows.lower, rows.upper, rows.step)):
762
+ return receiver, None
763
+ if isinstance(label, ast.Constant) and isinstance(label.value, str):
764
+ return receiver, label.value
765
+ return receiver, None
766
+
767
+ @staticmethod
768
+ def _column_names(node: ast.AST) -> tuple[str, ...] | None:
769
+ if isinstance(node, (ast.List, ast.Tuple)) and all(
770
+ isinstance(item, ast.Constant) and isinstance(item.value, str) for item in node.elts
771
+ ):
772
+ names = tuple(str(item.value) for item in node.elts if isinstance(item, ast.Constant))
773
+ return names if names and len(names) == len(set(names)) else None
774
+ return None
775
+
776
+ @staticmethod
777
+ def _combined(values: list[_Value], *, kind: Literal["series", "mask"] = "series") -> _Value:
778
+ indexes = {value.index_key for value in values if value.index_key is not None}
779
+ return _Value(
780
+ kind,
781
+ dependencies=tuple(dict.fromkeys(d for value in values for d in value.dependencies)),
782
+ evidence=_confidence(*(value.evidence for value in values)),
783
+ completeness=_completeness(*(value.completeness for value in values)),
784
+ epochs=tuple(dict.fromkeys(epoch for value in values for epoch in value.epochs)),
785
+ index_key=next(iter(indexes)) if len(indexes) == 1 else None,
786
+ history=tuple(dict.fromkeys(item for value in values for item in value.history)),
787
+ )
788
+
789
+ def _materialize(self, state: _State, context: _Context, node: ast.AST) -> None:
790
+ for name in sorted(self._known_columns(self._snapshot(state))):
791
+ self._read(state, name, context, node, record=False)
792
+
793
+ def _method_available(self, owner: str, name: str) -> bool:
794
+ symbol = f"pandas.{owner}.{name}"
795
+ return not any(
796
+ symbol == changed or symbol.startswith(changed + ".") for changed in self.api_mutations
797
+ )
798
+
799
+ def _expr(self, node: ast.AST, context: _Context) -> _Value:
800
+ value = self._expr_value(node, context)
801
+ if not isinstance(node, ast.Name) or not value.expression:
802
+ value = replace(value, expression=self._code(context, node))
803
+ return value
804
+
805
+ def _expr_value(self, node: ast.AST, context: _Context) -> _Value:
806
+ if self._literal(node):
807
+ return _Value("literal", completeness="complete")
808
+ if isinstance(node, ast.Name):
809
+ return self._refresh(context.env.get(node.id, _Value()))
810
+ if isinstance(node, ast.Attribute):
811
+ receiver = self._expr(node.value, context)
812
+ value = self._attribute(receiver, node.attr)
813
+ if context.key.startswith("template:") and receiver.symbol.startswith("module:"):
814
+ return self._isolated(value)
815
+ return value
816
+ if isinstance(node, ast.Subscript):
817
+ receiver_node, name = self._column_target(node)
818
+ receiver = self._expr(receiver_node, context)
819
+ if receiver.frame is not None:
820
+ if name is None:
821
+ if not (isinstance(node.value, ast.Attribute) and node.value.attr == "loc"):
822
+ names = (
823
+ (self._column_names(node.slice) if node.slice.elts else ())
824
+ if isinstance(node.slice, ast.List)
825
+ else None
826
+ )
827
+ if names is not None:
828
+ return self._project(receiver.frame, names, context, node)
829
+ mask = self._expr(node.slice, context)
830
+ if mask.kind == "mask" and mask.index_key == receiver.frame.index_key:
831
+ self._consume(mask, context, node, "filter")
832
+ self._diagnose(
833
+ context,
834
+ node,
835
+ "CT_ROW_CONTROL",
836
+ "Row selection changes control/cardinality; mask columns are "
837
+ "consumed but are not value dependencies of every output column.",
838
+ )
839
+ clone = self._clone(
840
+ receiver.frame, context, node, "filter", {}, receiver.evidence
841
+ )
842
+ clone.index_key = clone.record.id
843
+ self._materialize(clone, context, node)
844
+ return self._frame_value(clone)
845
+ self._diagnose(
846
+ context,
847
+ node,
848
+ "CT_DYNAMIC_COLUMN",
849
+ "Only a literal string column and full-row .loc are supported.",
850
+ )
851
+ return _Value(escaped=(receiver.frame.record.id,))
852
+ return self._read(receiver.frame, name, context, node)
853
+ if receiver.kind != "literal":
854
+ self._diagnose(
855
+ context,
856
+ node,
857
+ "CT_UNRESOLVED_RECEIVER",
858
+ "No resolved Pandas origin or DataFrame annotation for this receiver.",
859
+ )
860
+ return _Value(escaped=receiver.escaped)
861
+ if isinstance(node, ast.Compare):
862
+ values = [self._expr(node.left, context)] + [
863
+ self._expr(item, context) for item in node.comparators
864
+ ]
865
+ if (
866
+ all(
867
+ isinstance(op, (ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE))
868
+ for op in node.ops
869
+ )
870
+ and all(value.kind in {"literal", "series"} for value in values)
871
+ and any(value.kind == "series" for value in values)
872
+ ):
873
+ return self._combined(values, kind="mask")
874
+ return _Value(escaped=self._escaped(values))
875
+ if isinstance(node, ast.JoinedStr):
876
+ values = []
877
+ for formatted in node.values:
878
+ if isinstance(formatted, ast.FormattedValue):
879
+ values.append(self._expr(formatted.value, context))
880
+ if formatted.format_spec is not None:
881
+ values.append(self._expr(formatted.format_spec, context))
882
+ for value in values:
883
+ self._consume(value, context, node, "observe")
884
+ return replace(self._combined(values), kind="literal", index_key=None)
885
+ if isinstance(node, ast.BinOp) and isinstance(
886
+ node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow)
887
+ ):
888
+ left, right = self._expr(node.left, context), self._expr(node.right, context)
889
+ dependencies = tuple(dict.fromkeys(left.dependencies + right.dependencies))
890
+ known = left.kind in {"literal", "series"} and right.kind in {"literal", "series"}
891
+ return _Value(
892
+ "series" if known else "unknown",
893
+ dependencies=dependencies,
894
+ evidence=_confidence(left.evidence, right.evidence) if known else "unresolved",
895
+ completeness=_completeness(left.completeness, right.completeness),
896
+ epochs=tuple(dict.fromkeys(left.epochs + right.epochs)),
897
+ history=tuple(dict.fromkeys(left.history + right.history)),
898
+ )
899
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
900
+ return self._expr(node.operand, context)
901
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Invert):
902
+ value = self._expr(node.operand, context)
903
+ return value if value.kind == "mask" else _Value(escaped=self._escaped([value]))
904
+ if isinstance(node, ast.Call):
905
+ return self._call(node, context)
906
+ if isinstance(node, (ast.List, ast.Tuple, ast.Set, ast.Dict)):
907
+ values = [
908
+ self._expr(child, context)
909
+ for child in ast.iter_child_nodes(node)
910
+ if isinstance(child, ast.expr)
911
+ ]
912
+ for value in values:
913
+ self._consume(value, context, node, "container")
914
+ return _Value(escaped=self._escaped(values))
915
+ if isinstance(node, ast.Lambda):
916
+ return _Value()
917
+ self._diagnose(
918
+ context,
919
+ node,
920
+ "CT_UNSUPPORTED_EXPRESSION",
921
+ f"Expression {type(node).__name__} is outside the supported arithmetic subset.",
922
+ )
923
+ self._invalidate(list(context.env.values()), context, node)
924
+ return _Value()
925
+
926
+ def _attribute(self, receiver: _Value, name: str) -> _Value:
927
+ if receiver.frame is not None and name == "index":
928
+ return _Value(
929
+ "index",
930
+ index_key=receiver.frame.index_key,
931
+ completeness="complete",
932
+ escaped=(receiver.frame.record.id,),
933
+ )
934
+ if receiver.kind == "series" and name == "str":
935
+ return replace(receiver, kind="accessor")
936
+ if receiver.kind == "symbol":
937
+ symbol = f"{receiver.symbol}.{name}"
938
+ if any(
939
+ symbol == changed or symbol.startswith(changed + ".")
940
+ for changed in self.api_mutations
941
+ ):
942
+ return _Value()
943
+ if symbol.startswith("module:"):
944
+ return self._module_member(symbol.removeprefix("module:"))
945
+ return _Value("symbol", symbol=symbol)
946
+ return _Value(escaped=self._escaped([receiver]))
947
+
948
+ def _call(self, node: ast.Call, context: _Context) -> _Value:
949
+ if isinstance(node.func, ast.Attribute):
950
+ receiver = self._expr(node.func.value, context)
951
+ function = self._attribute(receiver, node.func.attr)
952
+ if receiver.frame is not None:
953
+ if node.func.attr == "assign" and self._method_available("DataFrame", "assign"):
954
+ return self._assign(node, receiver.frame, context)
955
+ if node.func.attr == "rename" and self._method_available("DataFrame", "rename"):
956
+ return self._rename(node, receiver.frame, context)
957
+ if node.func.attr == "dropna" and self._method_available("DataFrame", "dropna"):
958
+ return self._dropna(node, receiver.frame, context)
959
+ if node.func.attr in {"to_parquet", "to_csv"} and self._method_available(
960
+ "DataFrame", node.func.attr
961
+ ):
962
+ return self._sink(node, receiver, context)
963
+ if receiver.kind == "series":
964
+ if node.func.attr == "apply" and self._method_available("Series", "apply"):
965
+ return self._apply(node, receiver, context)
966
+ if (
967
+ node.func.attr == "tolist"
968
+ and not node.args
969
+ and not node.keywords
970
+ and (
971
+ receiver.components is not None
972
+ and self._method_available("Series", "tolist")
973
+ )
974
+ ):
975
+ return replace(receiver, kind="rows")
976
+ if receiver.kind == "accessor" and self._method_available("Series", "str"):
977
+ value = self._string_method(node, receiver, context)
978
+ if value is not None:
979
+ return value
980
+ else:
981
+ receiver = _Value()
982
+ function = self._expr(node.func, context)
983
+ symbol = function.symbol
984
+ arguments = [self._expr(arg, context) for arg in node.args]
985
+ keywords = [(keyword.arg, self._expr(keyword.value, context)) for keyword in node.keywords]
986
+ values = arguments + [value for _, value in keywords]
987
+ if (
988
+ isinstance(node.func, ast.Name)
989
+ and node.func.id not in context.env
990
+ and node.func.id in {"print", "len"}
991
+ and not any(isinstance(arg, ast.Starred) for arg in node.args)
992
+ and not any(name is None for name, _ in keywords)
993
+ ):
994
+ valid = (node.func.id == "len" and len(arguments) == 1 and not keywords) or (
995
+ node.func.id == "print"
996
+ and all(
997
+ name in {"sep", "end", "flush"} and value.kind == "literal"
998
+ for name, value in keywords
999
+ )
1000
+ )
1001
+ if valid:
1002
+ for value in values:
1003
+ self._consume(value, context, node, "observe")
1004
+ return _Value(
1005
+ "literal", completeness="partial" if node.func.id == "len" else "complete"
1006
+ )
1007
+ if receiver.frame is not None or receiver.dependencies or receiver.escaped:
1008
+ values.append(receiver)
1009
+ if symbol in {"pandas.read_csv", "pandas.read_parquet", "pandas.DataFrame"}:
1010
+ if symbol == "pandas.DataFrame":
1011
+ rows_frame = self._rows_frame(node, context, arguments, keywords)
1012
+ if rows_frame is not None:
1013
+ return rows_frame
1014
+ return self._source(node, symbol, context, values)
1015
+ if function.function is not None:
1016
+ return self._invoke(function.function, node, context, arguments, keywords)
1017
+ if function.escaped:
1018
+ values.append(function)
1019
+ for value in values:
1020
+ self._consume(value, context, node, "opaque_call")
1021
+ if any(value.frame is not None or value.dependencies or value.escaped for value in values):
1022
+ self._diagnose(
1023
+ context,
1024
+ node,
1025
+ "CT_OPAQUE_CALL",
1026
+ "Call effects and return provenance are unresolved; affected state is invalidated.",
1027
+ )
1028
+ self._invalidate(values, context, node)
1029
+ return _Value(escaped=self._escaped(values))
1030
+
1031
+ def _source(
1032
+ self, node: ast.Call, symbol: str, context: _Context, values: list[_Value]
1033
+ ) -> _Value:
1034
+ invalid = any(isinstance(arg, ast.Starred) for arg in node.args) or any(
1035
+ keyword.arg is None for keyword in node.keywords
1036
+ )
1037
+ if symbol == "pandas.read_csv":
1038
+ invalid |= len(node.args) > 1 or any(
1039
+ (
1040
+ kw.arg == "chunksize"
1041
+ and not (isinstance(kw.value, ast.Constant) and kw.value.value is None)
1042
+ )
1043
+ or (
1044
+ kw.arg == "iterator"
1045
+ and not (isinstance(kw.value, ast.Constant) and kw.value.value is False)
1046
+ )
1047
+ for kw in node.keywords
1048
+ )
1049
+ data: ast.AST | None = node.args[0] if len(node.args) == 1 else None
1050
+ if symbol == "pandas.DataFrame":
1051
+ for keyword in node.keywords:
1052
+ if keyword.arg == "data" and not node.args:
1053
+ data = keyword.value
1054
+ invalid |= (
1055
+ len(node.args) > 1
1056
+ or any(kw.arg != "data" for kw in node.keywords)
1057
+ or not isinstance(data, ast.Dict)
1058
+ or not self._literal(data)
1059
+ or any(
1060
+ not isinstance(key, ast.Constant) or not isinstance(key.value, str)
1061
+ for key in data.keys
1062
+ )
1063
+ )
1064
+ if invalid:
1065
+ self._diagnose(
1066
+ context,
1067
+ node,
1068
+ "CT_UNSUPPORTED_SOURCE",
1069
+ "This Pandas constructor/reader signature is outside the supported subset.",
1070
+ )
1071
+ self._invalidate(values, context, node)
1072
+ return _Value()
1073
+ state = self._new_frame(context, node, symbol)
1074
+ resolved = [value.expression for value in values if value.expression]
1075
+ expression = self._code(context, node)
1076
+ if resolved:
1077
+ expression += "\nArguments: " + ", ".join(resolved)
1078
+ state.history = (
1079
+ self._event(
1080
+ context,
1081
+ node,
1082
+ "source",
1083
+ f"Create DataFrame with {symbol}",
1084
+ frames=(state.record.id,),
1085
+ expression=expression,
1086
+ parents=context.parent_history,
1087
+ evidence=state.record.evidence,
1088
+ completeness="complete" if symbol == "pandas.DataFrame" else "partial",
1089
+ ),
1090
+ )
1091
+ if isinstance(data, ast.Dict) and symbol == "pandas.DataFrame":
1092
+ state.schema = set()
1093
+ for key in data.keys:
1094
+ assert isinstance(key, ast.Constant) and isinstance(key.value, str)
1095
+ self._write(
1096
+ state,
1097
+ key.value,
1098
+ _Value("literal", completeness="complete"),
1099
+ context,
1100
+ key,
1101
+ "source",
1102
+ )
1103
+ return self._frame_value(state)
1104
+
1105
+ def _apply(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value:
1106
+ arguments = [self._expr(arg, context) for arg in node.args]
1107
+ keywords = [self._expr(keyword.value, context) for keyword in node.keywords]
1108
+ callback = arguments[0].function if len(arguments) == 1 and not node.keywords else None
1109
+ symbols: dict[str, str] = {}
1110
+ if callback is not None:
1111
+ for name, binding in callback.module.env.items():
1112
+ symbol = binding.symbol if binding.kind == "symbol" else "<unresolved>"
1113
+ if any(
1114
+ symbol == changed or symbol.startswith(changed + ".")
1115
+ for changed in self.api_mutations
1116
+ ):
1117
+ symbol = "<unresolved>"
1118
+ symbols[name] = symbol
1119
+ summary = summarize_callback(callback.node, symbols=symbols) if callback else None
1120
+ if (
1121
+ summary is None
1122
+ or not summary.supported
1123
+ or summary.may_mutate
1124
+ or summary.components is None
1125
+ ):
1126
+ values = [receiver, *arguments, *keywords]
1127
+ if callback is not None and not context.key.startswith("template:"):
1128
+ values.extend(callback.module.env.values())
1129
+ for value in values:
1130
+ self._consume(value, context, node, "opaque_call")
1131
+ self._diagnose(
1132
+ context,
1133
+ node,
1134
+ "CT_UNSUPPORTED_APPLY",
1135
+ "Series.apply requires a resolved scalar callback with supported, fixed-shape "
1136
+ "returns and no unresolved mutation effects.",
1137
+ )
1138
+ self._invalidate(values, context, node)
1139
+ return _Value(escaped=self._escaped(values))
1140
+ assert callback is not None
1141
+ self._consume(receiver, context, node, "apply_input")
1142
+ callback_context = _Context(
1143
+ callback.module,
1144
+ callback.node.name,
1145
+ f"{context.key}>apply:{callback.module.name}.{callback.node.name}",
1146
+ {},
1147
+ context.stack,
1148
+ )
1149
+ self._consume(receiver, callback_context, callback.node, "callback_input")
1150
+ apply_event = self._event(
1151
+ context,
1152
+ node,
1153
+ "apply",
1154
+ f"Apply {callback.module.name}.{callback.node.name}",
1155
+ frames=self._escaped([receiver]),
1156
+ inputs=receiver.dependencies,
1157
+ parents=self._parents(receiver),
1158
+ evidence="inferred",
1159
+ completeness=receiver.completeness,
1160
+ )
1161
+ step_events: dict[str, str] = {}
1162
+ for step in summary.steps:
1163
+ marker = ast.Name(id="callback_step", ctx=ast.Load())
1164
+ marker.lineno, marker.col_offset = step.line, step.column
1165
+ marker.end_lineno, marker.end_col_offset = step.end_line, step.end_column
1166
+ step_events[step.id] = self._event(
1167
+ callback_context,
1168
+ marker,
1169
+ {
1170
+ "value": "callback_step",
1171
+ "control": "callback_control",
1172
+ "configuration": "callback_configuration",
1173
+ "context": "callback_context",
1174
+ }[step.role],
1175
+ step.label,
1176
+ frames=self._escaped([receiver]),
1177
+ inputs=receiver.dependencies if step.depends_on_input else (),
1178
+ parents=tuple(step_events[parent] for parent in step.parents) or (apply_event,),
1179
+ expression=step.expression,
1180
+ evidence="inferred",
1181
+ completeness=receiver.completeness,
1182
+ )
1183
+ self._diagnose(
1184
+ context,
1185
+ node,
1186
+ "CT_CALLBACK_INFERRED",
1187
+ f"Callback {callback.module.name}.{callback.node.name} resolved statically; "
1188
+ "scalar API behavior is inferred and runtime execution is not validated.",
1189
+ )
1190
+ components = tuple(
1191
+ _Value(
1192
+ "series",
1193
+ dependencies=receiver.dependencies if part.depends_on_input else (),
1194
+ evidence=_confidence(receiver.evidence, part.evidence),
1195
+ completeness=_completeness(receiver.completeness, part.completeness),
1196
+ epochs=receiver.epochs,
1197
+ index_key=receiver.index_key,
1198
+ producer="apply",
1199
+ expression=f"{self._code(context, node)} [return component {index + 1}]",
1200
+ history=tuple(step_events[item] for item in part.steps) or (apply_event,),
1201
+ )
1202
+ for index, part in enumerate(summary.components)
1203
+ )
1204
+ result = self._combined(list(components))
1205
+ return replace(result, components=components, producer="apply")
1206
+
1207
+ def _string_method(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value | None:
1208
+ assert isinstance(node.func, ast.Attribute)
1209
+ method = node.func.attr
1210
+ if not self._method_available("Series.str", method):
1211
+ return None
1212
+ no_args = {"strip", "lstrip", "rstrip", "lower", "upper", "casefold", "len"}
1213
+ if method not in no_args | {"contains", "startswith", "endswith", "replace"}:
1214
+ return None
1215
+ if not all(self._literal(arg) for arg in node.args) or not all(
1216
+ keyword.arg is not None and self._literal(keyword.value) for keyword in node.keywords
1217
+ ):
1218
+ return None
1219
+ if method in no_args and (node.args or node.keywords):
1220
+ return None
1221
+ self._consume(receiver, context, node, "string_transform")
1222
+ event = self._event(
1223
+ context,
1224
+ node,
1225
+ "string_transform",
1226
+ f"Series.str.{method}",
1227
+ frames=self._escaped([receiver]),
1228
+ inputs=receiver.dependencies,
1229
+ parents=self._parents(receiver),
1230
+ evidence=receiver.evidence,
1231
+ completeness=receiver.completeness,
1232
+ )
1233
+ return replace(
1234
+ receiver,
1235
+ kind="mask" if method in {"contains", "startswith", "endswith"} else "series",
1236
+ components=None,
1237
+ producer="derive",
1238
+ history=(event,),
1239
+ )
1240
+
1241
+ def _rows_frame(
1242
+ self,
1243
+ node: ast.Call,
1244
+ context: _Context,
1245
+ arguments: list[_Value],
1246
+ keywords: list[tuple[str | None, _Value]],
1247
+ ) -> _Value | None:
1248
+ options = dict(keywords)
1249
+ data = arguments[0] if len(arguments) == 1 else options.get("data", _Value())
1250
+ if data.kind != "rows":
1251
+ return None
1252
+ index = options.get("index", _Value())
1253
+ valid = (
1254
+ len(arguments) <= 1
1255
+ and not (arguments and "data" in options)
1256
+ and all(name in {"data", "index"} for name, _ in keywords)
1257
+ and data.components is not None
1258
+ and index.kind == "index"
1259
+ and data.index_key is not None
1260
+ and data.index_key == index.index_key
1261
+ )
1262
+ if not valid:
1263
+ self._diagnose(
1264
+ context,
1265
+ node,
1266
+ "CT_UNRESOLVED_ALIGNMENT",
1267
+ "Row-result DataFrame needs fixed width and an explicit matching source index.",
1268
+ )
1269
+ return _Value(escaped=self._escaped([data, index]))
1270
+ state = self._new_frame(context, node, "pandas.DataFrame.rows", data.evidence)
1271
+ state.positional = data.components
1272
+ state.schema = set()
1273
+ assert data.index_key is not None
1274
+ state.index_key = data.index_key
1275
+ return self._frame_value(state)
1276
+
1277
+ def _sink(self, node: ast.Call, receiver: _Value, context: _Context) -> _Value:
1278
+ assert receiver.frame is not None
1279
+ values = [self._expr(arg, context) for arg in node.args]
1280
+ values.extend(self._expr(keyword.value, context) for keyword in node.keywords)
1281
+ valid = len(node.args) <= 1 and all(
1282
+ keyword.arg is not None and self._literal(keyword.value) for keyword in node.keywords
1283
+ )
1284
+ self._materialize(receiver.frame, context, node)
1285
+ self._consume(receiver, context, node, "sink" if valid else "opaque_call")
1286
+ if valid:
1287
+ destinations = [value.expression for value in values if value.expression]
1288
+ self._event(
1289
+ context,
1290
+ node,
1291
+ "sink",
1292
+ "Export DataFrame" + (" to " + destinations[0] if destinations else ""),
1293
+ frames=(receiver.frame.record.id,),
1294
+ inputs=tuple(receiver.frame.current.values()),
1295
+ parents=receiver.frame.history,
1296
+ expression=self._code(context, node)
1297
+ + ("\nArguments: " + ", ".join(destinations) if destinations else ""),
1298
+ evidence=receiver.evidence,
1299
+ completeness="partial" if receiver.frame.schema is None else "complete",
1300
+ )
1301
+ return _Value("literal", completeness="complete")
1302
+ values.append(receiver)
1303
+ if not context.key.startswith("template:"):
1304
+ for value in list(values):
1305
+ if value.function is not None:
1306
+ values.extend(value.function.module.env.values())
1307
+ self._diagnose(
1308
+ context,
1309
+ node,
1310
+ "CT_UNSUPPORTED_SINK",
1311
+ "Export needs literal keyword options; dynamic options may have unknown effects.",
1312
+ )
1313
+ self._invalidate(values, context, node)
1314
+ return _Value(escaped=self._escaped(values))
1315
+
1316
+ def _project(
1317
+ self, state: _State, names: tuple[str, ...], context: _Context, node: ast.AST
1318
+ ) -> _Value:
1319
+ if any(
1320
+ name in state.removed or (state.schema is not None and name not in state.schema)
1321
+ for name in names
1322
+ ):
1323
+ self._diagnose(
1324
+ context, node, "CT_MISSING_COLUMN", "Projection contains a missing column."
1325
+ )
1326
+ return _Value()
1327
+ clone = self._clone(state, context, node, "projection", {}, state.record.evidence)
1328
+ clone.schema = set(names)
1329
+ # Record the structural selection even when no retained column creates
1330
+ # a pass event. An empty projection has a meaningful column fate.
1331
+ projection = self._event(
1332
+ context,
1333
+ node,
1334
+ "projection",
1335
+ "Project literal columns",
1336
+ frames=(state.record.id, clone.record.id),
1337
+ parents=state.history,
1338
+ evidence=state.record.evidence,
1339
+ completeness="complete",
1340
+ )
1341
+ clone.history = (projection,)
1342
+ for name in names:
1343
+ self._read(clone, name, context, node, record=False)
1344
+ return self._frame_value(clone)
1345
+
1346
+ def _dropna(self, node: ast.Call, state: _State, context: _Context) -> _Value:
1347
+ valid = not node.args
1348
+ subset: tuple[str, ...] | None = None
1349
+ inplace = False
1350
+ for keyword in node.keywords:
1351
+ if keyword.arg == "subset":
1352
+ subset = self._column_names(keyword.value)
1353
+ valid &= subset is not None
1354
+ elif keyword.arg == "inplace" and isinstance(keyword.value, ast.Constant):
1355
+ valid &= type(keyword.value.value) is bool
1356
+ inplace = keyword.value.value is True
1357
+ elif keyword.arg == "axis" and isinstance(keyword.value, ast.Constant):
1358
+ valid &= keyword.value.value in {0, "index"}
1359
+ elif keyword.arg == "how" and isinstance(keyword.value, ast.Constant):
1360
+ valid &= keyword.value.value in {"any", "all"}
1361
+ else:
1362
+ valid = False
1363
+ if not valid:
1364
+ for arg in node.args:
1365
+ self._expr(arg, context)
1366
+ for keyword in node.keywords:
1367
+ self._expr(keyword.value, context)
1368
+ self._diagnose(
1369
+ context,
1370
+ node,
1371
+ "CT_UNSUPPORTED_DROPNA",
1372
+ "Only row dropna with literal options is supported.",
1373
+ )
1374
+ self._invalidate([self._frame_value(state)], context, node)
1375
+ return _Value()
1376
+ for name in subset or sorted(self._known_columns(self._snapshot(state))):
1377
+ self._consume(
1378
+ self._read(state, name, context, node, record=False), context, node, "filter"
1379
+ )
1380
+ self._diagnose(
1381
+ context,
1382
+ node,
1383
+ "CT_ROW_CONTROL",
1384
+ "Dropna changes row control/cardinality; subset columns are recorded as filter "
1385
+ "consumptions without all-to-all value dependencies.",
1386
+ )
1387
+ if inplace:
1388
+ state.index_key = _identifier("index", state.index_key, self._location(context, node))
1389
+ return _Value("literal", completeness="complete")
1390
+ clone = self._clone(state, context, node, "dropna", {}, state.record.evidence)
1391
+ clone.index_key = clone.record.id
1392
+ self._materialize(clone, context, node)
1393
+ return self._frame_value(clone)
1394
+
1395
+ def _clone(
1396
+ self,
1397
+ state: _State,
1398
+ context: _Context,
1399
+ node: ast.AST,
1400
+ kind: str,
1401
+ names: dict[str, str],
1402
+ evidence: Confidence,
1403
+ ) -> _State:
1404
+ clone = self._new_frame(context, node, kind, _confidence(state.record.evidence, evidence))
1405
+ clone.schema = (
1406
+ None
1407
+ if state.schema is None
1408
+ else {
1409
+ next((new for new, old in names.items() if old == name), name)
1410
+ for name in state.schema
1411
+ }
1412
+ )
1413
+ clone.fallback = _Copy(self._snapshot(state), names, context, node, clone.record.evidence)
1414
+ clone.history = state.history
1415
+ clone.index_key = state.index_key
1416
+ clone.removed = (state.removed | set(names.values())) - names.keys()
1417
+ return clone
1418
+
1419
+ def _known_columns(self, snapshot: _Snapshot) -> set[str]:
1420
+ known = set(snapshot.current)
1421
+ if snapshot.fallback is not None:
1422
+ names = snapshot.fallback.names
1423
+ known.update(
1424
+ next((new for new, old in names.items() if old == name), name)
1425
+ for name in self._known_columns(snapshot.fallback.source)
1426
+ )
1427
+ known -= snapshot.removed
1428
+ return known if snapshot.schema is None else known & snapshot.schema
1429
+
1430
+ def _rename(self, node: ast.Call, state: _State, context: _Context) -> _Value:
1431
+ mapping: dict[str, str] = {}
1432
+ valid = not node.args
1433
+ found = False
1434
+ inplace = False
1435
+ for keyword in node.keywords:
1436
+ if keyword.arg == "columns" and isinstance(keyword.value, ast.Dict):
1437
+ found = True
1438
+ for key, value in zip(keyword.value.keys, keyword.value.values, strict=True):
1439
+ if (
1440
+ isinstance(key, ast.Constant)
1441
+ and isinstance(key.value, str)
1442
+ and isinstance(value, ast.Constant)
1443
+ and isinstance(value.value, str)
1444
+ ):
1445
+ mapping[key.value] = value.value
1446
+ else:
1447
+ valid = False
1448
+ elif keyword.arg == "inplace" and isinstance(keyword.value, ast.Constant):
1449
+ valid &= type(keyword.value.value) is bool
1450
+ inplace = keyword.value.value is True
1451
+ else:
1452
+ valid = False
1453
+ valid &= found and len(set(mapping.values())) == len(mapping)
1454
+ known = self._known_columns(self._snapshot(state))
1455
+ valid &= not any(new in known - mapping.keys() for new in mapping.values())
1456
+ valid &= not any(old in state.removed for old in mapping)
1457
+ if state.schema is not None:
1458
+ valid &= all(name in state.schema for name in mapping)
1459
+ valid &= not any(new in state.schema - mapping.keys() for new in mapping.values())
1460
+ if not valid:
1461
+ for arg in node.args:
1462
+ self._expr(arg, context)
1463
+ for keyword in node.keywords:
1464
+ self._expr(keyword.value, context)
1465
+ self._diagnose(
1466
+ context,
1467
+ node,
1468
+ "CT_UNSUPPORTED_RENAME",
1469
+ "Rename needs an injective literal columns mapping, no known schema collision, "
1470
+ "and a literal boolean inplace option.",
1471
+ )
1472
+ if any(keyword.arg == "inplace" for keyword in node.keywords):
1473
+ self._invalidate([self._frame_value(state)], context, node)
1474
+ return _Value()
1475
+ evidence: Confidence = "confirmed" if state.schema is not None else "inferred"
1476
+ if evidence == "inferred":
1477
+ self._diagnose(
1478
+ context,
1479
+ node,
1480
+ "CT_RENAME_SCHEMA_ASSUMPTION",
1481
+ "Rename assumes source columns exist and destination labels do not collide; "
1482
+ "the external schema is unknown.",
1483
+ )
1484
+ if inplace:
1485
+ before = self._snapshot(state)
1486
+ values: dict[str, _Value] = {}
1487
+ for old, new in mapping.items():
1488
+ source = self._lookup_snapshot(before, old, context, node)
1489
+ if source is not None:
1490
+ column = self.columns[source]
1491
+ values[new] = _Value(
1492
+ "series",
1493
+ dependencies=(source,),
1494
+ evidence=_confidence(evidence, column.evidence),
1495
+ completeness=column.completeness,
1496
+ )
1497
+ for old in mapping:
1498
+ state.current.pop(old, None)
1499
+ state.removed = (state.removed | set(mapping)) - set(mapping.values())
1500
+ if state.schema is not None:
1501
+ state.schema = {mapping.get(name, name) for name in state.schema}
1502
+ for new, column_value in values.items():
1503
+ self._write(state, new, column_value, context, node, "rename")
1504
+ return _Value("literal", completeness="complete")
1505
+ clone = self._clone(
1506
+ state, context, node, "rename", {new: old for old, new in mapping.items()}, evidence
1507
+ )
1508
+ for name in mapping.values():
1509
+ self._read(clone, name, context, node, record=False)
1510
+ return self._frame_value(clone)
1511
+
1512
+ def _assign(self, node: ast.Call, state: _State, context: _Context) -> _Value:
1513
+ if node.args or any(keyword.arg is None for keyword in node.keywords):
1514
+ self._diagnose(
1515
+ context,
1516
+ node,
1517
+ "CT_UNSUPPORTED_ASSIGN",
1518
+ "Assign requires explicit keyword arguments.",
1519
+ )
1520
+ self._invalidate([self._frame_value(state)], context, node)
1521
+ return _Value()
1522
+ eager: dict[str, _Value] = {}
1523
+ for keyword in node.keywords:
1524
+ assert keyword.arg is not None
1525
+ if not isinstance(keyword.value, ast.Lambda):
1526
+ eager[keyword.arg] = self._expr(keyword.value, context)
1527
+ clone = self._clone(state, context, node, "assign", {}, state.record.evidence)
1528
+ for keyword in node.keywords:
1529
+ assert keyword.arg is not None
1530
+ if isinstance(keyword.value, ast.Lambda):
1531
+ function = keyword.value
1532
+ args = function.args
1533
+ if (
1534
+ len(args.posonlyargs + args.args) != 1
1535
+ or args.kwonlyargs
1536
+ or args.vararg
1537
+ or args.kwarg
1538
+ or args.defaults
1539
+ ):
1540
+ self._diagnose(
1541
+ context,
1542
+ function,
1543
+ "CT_UNSUPPORTED_ASSIGN",
1544
+ "Assign lambdas need one positional parameter and no defaults.",
1545
+ )
1546
+ value = _Value()
1547
+ else:
1548
+ env = dict(context.env)
1549
+ env[(args.posonlyargs + args.args)[0].arg] = self._frame_value(clone)
1550
+ nested = _Context(
1551
+ context.module, context.scope, context.key, env, context.stack
1552
+ )
1553
+ value = self._expr(function.body, nested)
1554
+ else:
1555
+ value = eager[keyword.arg]
1556
+ if value.kind not in {"literal", "series"}:
1557
+ self._diagnose(
1558
+ context,
1559
+ keyword.value,
1560
+ "CT_UNSUPPORTED_ASSIGN",
1561
+ "An unresolved assign value may be callable; its effects on the copy "
1562
+ "are not modeled.",
1563
+ )
1564
+ self._invalidate([self._frame_value(clone)], context, keyword.value)
1565
+ self._write(clone, keyword.arg, value, context, keyword.value)
1566
+ return self._frame_value(clone)
1567
+
1568
+ def _module_member(self, dotted: str, *, prefer_attribute: bool = True) -> _Value:
1569
+ if self._ambiguous_module(dotted):
1570
+ return _Value()
1571
+ module_name, _, member = dotted.rpartition(".")
1572
+ parent = self.modules.get(module_name)
1573
+ if parent is not None:
1574
+ self._module(parent)
1575
+ if prefer_attribute and member in parent.env:
1576
+ return parent.env[member]
1577
+ if dotted in self.modules:
1578
+ self._module(self.modules[dotted])
1579
+ value = _Value("symbol", symbol=f"module:{dotted}")
1580
+ if parent is not None:
1581
+ parent.env[member] = value
1582
+ return value
1583
+ if parent is not None:
1584
+ return parent.env.get(member, _Value())
1585
+ return _Value("symbol", symbol=f"module:{dotted}")
1586
+
1587
+ def _ambiguous_module(self, name: str) -> bool:
1588
+ return any(name == module or name.startswith(module + ".") for module in self.ambiguous)
1589
+
1590
+ def _import(self, node: ast.Import | ast.ImportFrom, context: _Context) -> None:
1591
+ if isinstance(node, ast.Import):
1592
+ for alias in node.names:
1593
+ name = alias.asname or alias.name.split(".")[0]
1594
+ if self._ambiguous_module(alias.name):
1595
+ context.env[name] = _Value()
1596
+ continue
1597
+ if alias.name in self.modules or any(
1598
+ key.startswith(alias.name + ".") for key in self.modules
1599
+ ):
1600
+ self._module_member(alias.name, prefer_attribute=False)
1601
+ module_name = alias.name if alias.asname else alias.name.split(".")[0]
1602
+ context.env[name] = _Value("symbol", symbol=f"module:{module_name}")
1603
+ else:
1604
+ context.env[name] = _Value("symbol", symbol=alias.name)
1605
+ return
1606
+ module_name = node.module or ""
1607
+ if node.level:
1608
+ package = context.module.name.split(".")
1609
+ if not context.module.path.endswith("/__init__.py"):
1610
+ package = package[:-1]
1611
+ if node.level > len(package):
1612
+ module_name = ""
1613
+ else:
1614
+ module_name = ".".join(
1615
+ package[: len(package) - node.level + 1]
1616
+ + (module_name.split(".") if module_name else [])
1617
+ )
1618
+ for alias in node.names:
1619
+ if alias.name == "*":
1620
+ self._diagnose(
1621
+ context, node, "CT_STAR_IMPORT", "Wildcard imports invalidate name resolution."
1622
+ )
1623
+ self._invalidate(list(context.env.values()), context, node)
1624
+ context.env.clear()
1625
+ continue
1626
+ if self._ambiguous_module(module_name):
1627
+ context.env[alias.asname or alias.name] = _Value()
1628
+ continue
1629
+ if module_name in self.modules or any(
1630
+ key.startswith(module_name + ".") for key in self.modules
1631
+ ):
1632
+ value = self._module_member(f"{module_name}.{alias.name}")
1633
+ else:
1634
+ value = _Value("symbol", symbol=f"{module_name}.{alias.name}")
1635
+ if value.symbol in self.api_mutations:
1636
+ value = _Value()
1637
+ if context.key.startswith("template:"):
1638
+ value = self._isolated(value)
1639
+ context.env[alias.asname or alias.name] = value
1640
+
1641
+ def _bind(self, target: ast.AST, value: _Value, context: _Context) -> None:
1642
+ if isinstance(target, ast.Name):
1643
+ context.env[target.id] = value
1644
+ if value.frame is not None:
1645
+ frame = value.frame
1646
+ event = self._event(
1647
+ context,
1648
+ target,
1649
+ "bind",
1650
+ f"{target.id} <- {value.expression or frame.record.origin}",
1651
+ frames=(frame.record.id,),
1652
+ parents=tuple(dict.fromkeys(frame.history + value.history)),
1653
+ expression=f"{target.id} = {value.expression}"
1654
+ if value.expression
1655
+ else target.id,
1656
+ evidence=value.evidence,
1657
+ completeness=value.completeness,
1658
+ )
1659
+ frame.history = (event,)
1660
+ context.env[target.id] = replace(value, history=(event,))
1661
+ return
1662
+ if isinstance(target, ast.Subscript):
1663
+ receiver_node, name = self._column_target(target)
1664
+ receiver = self._expr(receiver_node, context)
1665
+ if receiver.frame is not None:
1666
+ if name is not None:
1667
+ self._write(receiver.frame, name, value, context, target)
1668
+ else:
1669
+ names = (
1670
+ self._column_names(target.slice)
1671
+ if isinstance(target.slice, ast.List)
1672
+ else None
1673
+ )
1674
+ if names is not None:
1675
+ source = value.frame
1676
+ parts = source.positional if source is not None else None
1677
+ if (
1678
+ parts is not None
1679
+ and len(parts) == len(names)
1680
+ and source is not None
1681
+ and source.index_key == receiver.frame.index_key
1682
+ ):
1683
+ for column_name, part in zip(names, parts, strict=True):
1684
+ self._write(receiver.frame, column_name, part, context, target)
1685
+ return
1686
+ self._diagnose(
1687
+ context,
1688
+ target,
1689
+ "CT_UNRESOLVED_MULTI_ASSIGN",
1690
+ "Multi-column assignment needs matching tuple width and row index; "
1691
+ "no positional dependencies are assumed.",
1692
+ )
1693
+ self._invalidate([receiver], context, target)
1694
+ return
1695
+ self._diagnose(
1696
+ context,
1697
+ target,
1698
+ "CT_DYNAMIC_COLUMN",
1699
+ "A dynamic or partial column write invalidates the DataFrame state.",
1700
+ )
1701
+ self._invalidate([receiver], context, target)
1702
+ elif receiver.kind != "literal":
1703
+ self._diagnose(
1704
+ context,
1705
+ target,
1706
+ "CT_UNRESOLVED_RECEIVER",
1707
+ "Cannot bind a column write to a resolved DataFrame.",
1708
+ )
1709
+ self._invalidate([receiver], context, target)
1710
+ return
1711
+ if isinstance(target, ast.Attribute):
1712
+ receiver = self._expr(target.value, context)
1713
+ symbol = self._symbol(target, context)
1714
+ if symbol:
1715
+ self.api_mutations.add(symbol)
1716
+ if receiver.frame is not None or receiver.dependencies or symbol.startswith("pandas"):
1717
+ self._diagnose(
1718
+ context,
1719
+ target,
1720
+ "CT_ATTRIBUTE_MUTATION",
1721
+ "Attribute mutation is not modeled and invalidates affected evidence.",
1722
+ )
1723
+ self._invalidate([receiver], context, target)
1724
+ return
1725
+ self._diagnose(
1726
+ context, target, "CT_UNSUPPORTED_TARGET", "Unpacking/chained targets are not modeled."
1727
+ )
1728
+ self._invalidate([value] + list(context.env.values()), context, target)
1729
+ for child in ast.walk(target):
1730
+ if isinstance(child, ast.Name):
1731
+ context.env[child.id] = _Value()
1732
+
1733
+ @staticmethod
1734
+ def _bound_names(node: ast.AST) -> set[str]:
1735
+ names: set[str] = set()
1736
+
1737
+ class Visitor(ast.NodeVisitor):
1738
+ def visit_Name(self, child: ast.Name) -> None:
1739
+ if isinstance(child.ctx, (ast.Store, ast.Del)):
1740
+ names.add(child.id)
1741
+
1742
+ def visit_FunctionDef(self, child: ast.FunctionDef) -> None:
1743
+ names.add(child.name)
1744
+
1745
+ def visit_AsyncFunctionDef(self, child: ast.AsyncFunctionDef) -> None:
1746
+ names.add(child.name)
1747
+
1748
+ def visit_ClassDef(self, child: ast.ClassDef) -> None:
1749
+ names.add(child.name)
1750
+
1751
+ def visit_Lambda(self, child: ast.Lambda) -> None:
1752
+ pass
1753
+
1754
+ def visit_Import(self, child: ast.Import) -> None:
1755
+ names.update(alias.asname or alias.name.split(".")[0] for alias in child.names)
1756
+
1757
+ def visit_ImportFrom(self, child: ast.ImportFrom) -> None:
1758
+ names.update(alias.asname or alias.name for alias in child.names)
1759
+
1760
+ def visit_ExceptHandler(self, child: ast.ExceptHandler) -> None:
1761
+ if child.name:
1762
+ names.add(child.name)
1763
+ self.generic_visit(child)
1764
+
1765
+ def visit_MatchAs(self, child: ast.MatchAs) -> None:
1766
+ if child.name:
1767
+ names.add(child.name)
1768
+ self.generic_visit(child)
1769
+
1770
+ def visit_MatchStar(self, child: ast.MatchStar) -> None:
1771
+ if child.name:
1772
+ names.add(child.name)
1773
+
1774
+ def visit_MatchMapping(self, child: ast.MatchMapping) -> None:
1775
+ if child.rest:
1776
+ names.add(child.rest)
1777
+ self.generic_visit(child)
1778
+
1779
+ visitor = Visitor()
1780
+ if isinstance(node, ast.FunctionDef):
1781
+ for statement in node.body:
1782
+ visitor.visit(statement)
1783
+ else:
1784
+ visitor.visit(node)
1785
+ return names
1786
+
1787
+ def _statements(self, statements: list[ast.stmt], context: _Context) -> _Value:
1788
+ for node in statements:
1789
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
1790
+ self._import(node, context)
1791
+ elif isinstance(node, ast.Assign):
1792
+ value = self._expr(node.value, context)
1793
+ for target in node.targets:
1794
+ self._bind(target, value, context)
1795
+ elif isinstance(node, ast.AnnAssign):
1796
+ if node.value is not None:
1797
+ self._bind(node.target, self._expr(node.value, context), context)
1798
+ elif isinstance(node, ast.Expr):
1799
+ self._expr(node.value, context)
1800
+ elif isinstance(node, ast.FunctionDef):
1801
+ for decorator in node.decorator_list:
1802
+ decoration = self._expr(decorator, context)
1803
+ self._invalidate([decoration], context, decorator)
1804
+ if decoration.function is not None and not context.key.startswith("template:"):
1805
+ self._diagnose(
1806
+ context,
1807
+ decorator,
1808
+ "CT_DECORATOR_EFFECTS",
1809
+ "Applying the resolved decorator may mutate module state; its globals "
1810
+ "are invalidated rather than assuming a pure decoration.",
1811
+ )
1812
+ self._invalidate(
1813
+ list(decoration.function.module.env.values()), context, decorator
1814
+ )
1815
+ arguments = node.args.posonlyargs + node.args.args
1816
+ default_nodes = list(
1817
+ zip(arguments[-len(node.args.defaults) :], node.args.defaults, strict=False)
1818
+ )
1819
+ defaults = {arg.arg: self._expr(default, context) for arg, default in default_nodes}
1820
+ defaults.update(
1821
+ {
1822
+ arg.arg: self._expr(default, context)
1823
+ for arg, default in zip(
1824
+ node.args.kwonlyargs, node.args.kw_defaults, strict=True
1825
+ )
1826
+ if default is not None
1827
+ }
1828
+ )
1829
+ annotations = [
1830
+ argument.annotation for argument in arguments + node.args.kwonlyargs
1831
+ ] + [node.returns]
1832
+ if node.args.vararg:
1833
+ annotations.append(node.args.vararg.annotation)
1834
+ if node.args.kwarg:
1835
+ annotations.append(node.args.kwarg.annotation)
1836
+ for annotation in annotations:
1837
+ if annotation is not None and not isinstance(
1838
+ annotation, (ast.Name, ast.Attribute, ast.Constant)
1839
+ ):
1840
+ # Annotation evaluation differs across Python versions
1841
+ # and future imports. Never execute it, and do not let
1842
+ # possible effects preserve confirmed object history.
1843
+ self._diagnose(
1844
+ context,
1845
+ annotation,
1846
+ "CT_UNSUPPORTED_ANNOTATION",
1847
+ "Dynamic annotation effects are unknown; tracked state is invalidated.",
1848
+ )
1849
+ self._invalidate(list(context.env.values()), context, annotation)
1850
+ if node.decorator_list or context.scope != "<module>":
1851
+ bound = self._bound_names(node) | {
1852
+ arg.arg for arg in arguments + node.args.kwonlyargs
1853
+ }
1854
+ captures = [
1855
+ context.env[child.id]
1856
+ for child in ast.walk(node)
1857
+ if isinstance(child, ast.Name)
1858
+ and isinstance(child.ctx, ast.Load)
1859
+ and child.id not in bound
1860
+ and child.id in context.env
1861
+ ]
1862
+ context.env[node.name] = _Value(escaped=self._escaped(captures))
1863
+ self._diagnose(
1864
+ context,
1865
+ node,
1866
+ "CT_UNSUPPORTED_FUNCTION",
1867
+ "Nested/decorated function body is unresolved; defining it does not "
1868
+ "execute its body. Defaults and decorators are analyzed separately.",
1869
+ )
1870
+ continue
1871
+ frame_parameters = {
1872
+ argument.arg
1873
+ for argument in arguments + node.args.kwonlyargs
1874
+ if self._dataframe_annotation(argument.annotation, context)
1875
+ }
1876
+ function = _Function(context.module, node, defaults, frame_parameters)
1877
+ self.functions.append(function)
1878
+ context.env[node.name] = _Value("function", function=function)
1879
+ elif isinstance(node, ast.Return):
1880
+ value = self._expr(node.value, context) if node.value else _Value()
1881
+ event = self._consume(value, context, node, "return")
1882
+ if event is not None and value.frame is not None:
1883
+ value.frame.history = (event,)
1884
+ value = replace(value, history=(event,))
1885
+ return value
1886
+ elif isinstance(node, (ast.Pass, ast.Assert)):
1887
+ if isinstance(node, ast.Assert):
1888
+ self._expr(node.test, context)
1889
+ self._diagnose(
1890
+ context,
1891
+ node,
1892
+ "CT_UNSUPPORTED_CONTROL_FLOW",
1893
+ "Assertions may terminate execution; reachability is not guaranteed.",
1894
+ )
1895
+ else:
1896
+ self._diagnose(
1897
+ context,
1898
+ node,
1899
+ "CT_UNSUPPORTED_CONTROL_FLOW",
1900
+ f"{type(node).__name__} is outside linear analysis; "
1901
+ "tracked state is invalidated.",
1902
+ )
1903
+ self._invalidate(list(context.env.values()), context, node)
1904
+ self._invalidate_api_writes(node, context)
1905
+ for name in self._bound_names(node):
1906
+ context.env[name] = _Value()
1907
+ return _Value()
1908
+
1909
+ def _invalidate_api_writes(self, node: ast.AST, context: _Context) -> None:
1910
+ for child in ast.walk(node):
1911
+ if isinstance(child, ast.Attribute) and isinstance(child.ctx, (ast.Store, ast.Del)):
1912
+ symbol = self._symbol(child, context)
1913
+ if symbol:
1914
+ self.api_mutations.add(symbol)
1915
+
1916
+ def _invoke(
1917
+ self,
1918
+ function: _Function,
1919
+ call: ast.Call,
1920
+ caller: _Context,
1921
+ positional: list[_Value],
1922
+ keywords: list[tuple[str | None, _Value]],
1923
+ *,
1924
+ template: bool = False,
1925
+ ) -> _Value:
1926
+ node = function.node
1927
+ symbolic = template or caller.key.startswith("template:")
1928
+ identity = (function.module.name, node.name)
1929
+ invalid = bool(node.args.vararg or node.args.kwarg)
1930
+ invalid |= identity in caller.stack or len(caller.stack) >= self.MAX_DEPTH
1931
+ invalid |= self.call_count >= self.MAX_CALLS
1932
+ invalid |= any(isinstance(arg, ast.Starred) for arg in call.args)
1933
+ invalid |= any(name is None for name, _ in keywords)
1934
+ invalid |= any(
1935
+ isinstance(child, (ast.Global, ast.Nonlocal, ast.Yield, ast.YieldFrom))
1936
+ for child in ast.walk(node)
1937
+ )
1938
+ arguments = node.args.posonlyargs + node.args.args
1939
+ names = [arg.arg for arg in arguments]
1940
+ values = dict(function.defaults)
1941
+ if symbolic:
1942
+ values = {name: self._isolated(value) for name, value in values.items()}
1943
+ invalid |= len(positional) > len(arguments)
1944
+ assigned: set[str] = set()
1945
+ for name, value in zip(names, positional, strict=False):
1946
+ values[name] = value
1947
+ assigned.add(name)
1948
+ for keyword_name, value in keywords:
1949
+ if keyword_name is None:
1950
+ continue
1951
+ invalid |= keyword_name in assigned or keyword_name in {
1952
+ arg.arg for arg in node.args.posonlyargs
1953
+ }
1954
+ invalid |= keyword_name not in names + [arg.arg for arg in node.args.kwonlyargs]
1955
+ values[keyword_name] = value
1956
+ assigned.add(keyword_name)
1957
+ invalid |= any(arg.arg not in values for arg in arguments + node.args.kwonlyargs)
1958
+ # Early returns cannot be flattened into a single execution path.
1959
+ invalid |= any(isinstance(statement, ast.Return) for statement in node.body[:-1])
1960
+ if invalid:
1961
+ self._diagnose(
1962
+ caller,
1963
+ call,
1964
+ "CT_UNRESOLVED_CALL",
1965
+ "Call signature, recursion, expansion budget, or function body is unsupported.",
1966
+ )
1967
+ inputs = positional + list(values.values())
1968
+ if not symbolic:
1969
+ # An unsupported body can mutate defaults or module objects
1970
+ # without receiving them as explicit call arguments.
1971
+ inputs.extend(function.module.env.values())
1972
+ for value in inputs:
1973
+ self._consume(value, caller, call, "opaque_call")
1974
+ self._invalidate(inputs, caller, call)
1975
+ effects_env = dict(function.module.env)
1976
+ for name in self._bound_names(node):
1977
+ effects_env[name] = _Value()
1978
+ effects_env.update(values)
1979
+ effects_context = _Context(
1980
+ function.module, node.name, caller.key, effects_env, caller.stack
1981
+ )
1982
+ self._invalidate_api_writes(node, effects_context)
1983
+ if not symbolic:
1984
+ for child in ast.walk(node):
1985
+ if isinstance(child, ast.Global):
1986
+ for name in child.names:
1987
+ value = function.module.env.get(name, _Value())
1988
+ self._invalidate([value])
1989
+ function.module.env[name] = _Value()
1990
+ return _Value()
1991
+ function.called = True
1992
+ self.call_count += 1
1993
+ env = dict(function.module.env)
1994
+ if symbolic:
1995
+ env = {name: self._isolated(value) for name, value in env.items()}
1996
+ for name in self._bound_names(node):
1997
+ env[name] = _Value()
1998
+ env.update(values)
1999
+ location = self._location(caller, call)
2000
+ key = (
2001
+ f"template:{function.module.name}.{node.name}"
2002
+ if template
2003
+ else f"{caller.key}>{function.module.name}.{node.name}@"
2004
+ f"{location.path}:{location.line}:{location.column}"
2005
+ )
2006
+ supplied = list(values.values())
2007
+ call_event = self._event(
2008
+ caller,
2009
+ call,
2010
+ "call",
2011
+ (
2012
+ f"Analyze function {function.module.name}.{node.name}"
2013
+ if template
2014
+ else f"Call {function.module.name}.{node.name}"
2015
+ ),
2016
+ frames=self._escaped(supplied),
2017
+ inputs=tuple(dict.fromkeys(item for value in supplied for item in value.dependencies)),
2018
+ parents=tuple(
2019
+ dict.fromkeys(
2020
+ caller.parent_history
2021
+ + tuple(item for value in supplied for item in self._parents(value))
2022
+ )
2023
+ ),
2024
+ expression=(
2025
+ "Symbolic function analysis; no resolved caller"
2026
+ if template
2027
+ else self._code(caller, call)
2028
+ )
2029
+ + "\nParameters: "
2030
+ + ", ".join(f"{name} <- {value.expression or name}" for name, value in values.items()),
2031
+ evidence="inferred" if symbolic else "confirmed",
2032
+ )
2033
+ context = _Context(
2034
+ function.module, node.name, key, env, caller.stack + (identity,), (call_event,)
2035
+ )
2036
+ for value in supplied:
2037
+ if value.frame is not None:
2038
+ value.frame.history = tuple(dict.fromkeys(value.frame.history + (call_event,)))
2039
+ for argument in (*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs):
2040
+ value = values[argument.arg]
2041
+ if value.frame is not None:
2042
+ # Preserve the proven parameter binding in the callee context;
2043
+ # identical parameter names in unrelated calls stay separate.
2044
+ target = ast.copy_location(ast.Name(id=argument.arg, ctx=ast.Store()), argument)
2045
+ self._bind(target, value, context)
2046
+ for value in positional + [value for _, value in keywords]:
2047
+ self._consume(value, caller, call, "argument")
2048
+ result = self._statements(node.body, context)
2049
+ if result.frame is not None:
2050
+ return result
2051
+ return _Value(escaped=self._escaped([result]))
2052
+
2053
+ def _module(self, module: _Module) -> None:
2054
+ if module.status != "new":
2055
+ return
2056
+ module.status = "visiting"
2057
+ context = _Context(module, "<module>", f"module:{module.name}", module.env)
2058
+ try:
2059
+ self._statements(module.tree.body, context)
2060
+ except RecursionError:
2061
+ self._diagnose(
2062
+ context, module.tree, "CT_ANALYSIS_LIMIT", "AST interpretation depth was exceeded."
2063
+ )
2064
+ self._invalidate(list(context.env.values()))
2065
+ module.status = "done"
2066
+
2067
+ def run(self) -> None:
2068
+ for module in self.modules.values():
2069
+ self._module(module)
2070
+ for function in list(self.functions):
2071
+ if function.called:
2072
+ continue
2073
+ module = function.module
2074
+ node = function.node
2075
+ context = _Context(module, node.name, f"template:{module.name}.{node.name}", module.env)
2076
+ values: list[_Value] = []
2077
+ keywords: list[tuple[str | None, _Value]] = []
2078
+ for argument in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
2079
+ if argument.arg in function.frame_parameters:
2080
+ state = self._new_frame(
2081
+ context, argument, f"parameter:{argument.arg}", "inferred"
2082
+ )
2083
+ value = self._frame_value(state)
2084
+ else:
2085
+ value = self._isolated(function.defaults.get(argument.arg, _Value()))
2086
+ if argument in node.args.kwonlyargs:
2087
+ keywords.append((argument.arg, value))
2088
+ else:
2089
+ values.append(value)
2090
+ call = ast.Call(func=ast.Name(id=node.name), args=[], keywords=[])
2091
+ ast.copy_location(call, node)
2092
+ mutations = set(self.api_mutations)
2093
+ try:
2094
+ self._invoke(function, call, context, values, keywords, template=True)
2095
+ except RecursionError:
2096
+ self._diagnose(
2097
+ context, node, "CT_ANALYSIS_LIMIT", "Function AST traversal depth was exceeded."
2098
+ )
2099
+ self._invalidate(values + [value for _, value in keywords])
2100
+ finally:
2101
+ self.api_mutations = mutations
2102
+
2103
+ @staticmethod
2104
+ def _isolated(value: _Value) -> _Value:
2105
+ if (
2106
+ value.frame is not None
2107
+ or value.dependencies
2108
+ or value.escaped
2109
+ or value.epochs
2110
+ or value.components is not None
2111
+ or value.index_key is not None
2112
+ ):
2113
+ return _Value()
2114
+ return value
2115
+
2116
+ def _dataframe_annotation(self, annotation: ast.AST | None, context: _Context) -> bool:
2117
+ if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str):
2118
+ try:
2119
+ annotation = ast.parse(annotation.value, mode="eval").body
2120
+ except (SyntaxError, RecursionError):
2121
+ return False
2122
+ return annotation is not None and self._symbol(annotation, context) == "pandas.DataFrame"
2123
+
2124
+
2125
+ def analyze_project(path: str | Path, *, config: ScanConfig | None = None) -> LineageReport:
2126
+ """Build static lineage for resolved, linear Pandas code without executing it.
2127
+
2128
+ ``scan_project`` remains available for the broader occurrence inventory. This
2129
+ report contains graph facts only where the lineage interpreter has evidence.
2130
+ """
2131
+ root = Path(path).resolve()
2132
+ discovery = scan_project(root, config=config)
2133
+ modules: dict[str, _Module] = {}
2134
+ ambiguous: set[str] = set()
2135
+ diagnostics = [
2136
+ diagnostic
2137
+ for diagnostic in discovery.diagnostics
2138
+ if diagnostic.code
2139
+ in {
2140
+ "CT_DISCOVERY_ERROR",
2141
+ "CT_READ_ERROR",
2142
+ "CT_ENCODING_ERROR",
2143
+ "CT_SYNTAX_ERROR",
2144
+ "CT_ANALYSIS_LIMIT",
2145
+ }
2146
+ ]
2147
+ limited_files = {item.location.path for item in diagnostics if item.code == "CT_ANALYSIS_LIMIT"}
2148
+ for file in discovery.files:
2149
+ if file.status != "parsed" or file.path in limited_files:
2150
+ continue
2151
+ try:
2152
+ raw = (root / file.path).read_bytes()
2153
+ if hashlib.sha256(raw).hexdigest() != file.sha256:
2154
+ raise ValueError("Source changed during analysis; rescan a stable project.")
2155
+ encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline)
2156
+ source = raw.decode(encoding).replace("\r\n", "\n").replace("\r", "\n")
2157
+ tree = ast.parse(source, filename=file.path)
2158
+ except (OSError, UnicodeError, LookupError, SyntaxError, RecursionError, ValueError) as exc:
2159
+ diagnostics.append(
2160
+ Diagnostic("CT_READ_ERROR", "error", str(exc), Location(file.path, 1, 0))
2161
+ )
2162
+ continue
2163
+ if file.module in modules or file.module in ambiguous:
2164
+ diagnostics.append(
2165
+ Diagnostic(
2166
+ "CT_AMBIGUOUS_MODULE",
2167
+ "error",
2168
+ "Multiple files resolve to this module name.",
2169
+ Location(file.path, 1, 0),
2170
+ )
2171
+ )
2172
+ ambiguous.add(file.module)
2173
+ modules.pop(file.module, None)
2174
+ continue
2175
+ modules[file.module] = _Module(file.module, file.path, source, tree)
2176
+ modules = {
2177
+ name: module
2178
+ for name, module in modules.items()
2179
+ if not any(name.startswith(prefix + ".") for prefix in ambiguous)
2180
+ }
2181
+ analyzer = _Analyzer(modules, ambiguous)
2182
+ analyzer.run()
2183
+ diagnostics.extend(analyzer.diagnostics)
2184
+ return LineageReport(
2185
+ files=discovery.files,
2186
+ functions=discovery.functions,
2187
+ dataframes=tuple(analyzer.frames),
2188
+ columns=tuple(analyzer.columns.values()),
2189
+ operations=tuple(analyzer.operations),
2190
+ edges=tuple(analyzer.edges),
2191
+ consumptions=tuple(analyzer.consumptions),
2192
+ history=tuple(analyzer.history),
2193
+ diagnostics=tuple(
2194
+ sorted(
2195
+ diagnostics,
2196
+ key=lambda diagnostic: (
2197
+ diagnostic.location.path,
2198
+ diagnostic.location.line,
2199
+ diagnostic.location.column,
2200
+ diagnostic.code,
2201
+ diagnostic.message,
2202
+ ),
2203
+ )
2204
+ ),
2205
+ exclude=discovery.exclude,
2206
+ )