coltrace 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coltrace/__init__.py +9 -0
- coltrace/__main__.py +6 -0
- coltrace/analysis.py +2206 -0
- coltrace/callbacks.py +579 -0
- coltrace/cli.py +668 -0
- coltrace/collector.py +561 -0
- coltrace/exporters.py +676 -0
- coltrace/graph.py +380 -0
- coltrace/history.py +441 -0
- coltrace/journey.py +578 -0
- coltrace/lineage_models.py +112 -0
- coltrace/models.py +92 -0
- coltrace/py.typed +0 -0
- coltrace/scanner.py +236 -0
- coltrace/templates/graph.html +418 -0
- coltrace/visual_layout.py +159 -0
- coltrace-0.1.0.dev4.dist-info/METADATA +241 -0
- coltrace-0.1.0.dev4.dist-info/RECORD +22 -0
- coltrace-0.1.0.dev4.dist-info/WHEEL +5 -0
- coltrace-0.1.0.dev4.dist-info/entry_points.txt +2 -0
- coltrace-0.1.0.dev4.dist-info/licenses/LICENSE +21 -0
- coltrace-0.1.0.dev4.dist-info/top_level.txt +1 -0
coltrace/exporters.py
ADDED
|
@@ -0,0 +1,676 @@
|
|
|
1
|
+
"""Validated JSON snapshots and self-contained, offline graph exports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import html
|
|
7
|
+
import json
|
|
8
|
+
import textwrap
|
|
9
|
+
import types
|
|
10
|
+
import warnings
|
|
11
|
+
from dataclasses import asdict, fields, is_dataclass
|
|
12
|
+
from importlib.resources import files
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any, Literal, get_args, get_origin, get_type_hints
|
|
15
|
+
|
|
16
|
+
import networkx as nx
|
|
17
|
+
|
|
18
|
+
from .graph import LineageGraph
|
|
19
|
+
from .lineage_models import LineageReport
|
|
20
|
+
from .models import Location
|
|
21
|
+
from .visual_layout import layout_history
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _decode(expected: Any, value: Any, where: str) -> Any:
|
|
25
|
+
origin = get_origin(expected)
|
|
26
|
+
if origin is Literal:
|
|
27
|
+
if value not in get_args(expected) or not isinstance(value, str):
|
|
28
|
+
raise ValueError(f"{where}: invalid enum value")
|
|
29
|
+
return value
|
|
30
|
+
if origin is types.UnionType:
|
|
31
|
+
for alternative in get_args(expected):
|
|
32
|
+
try:
|
|
33
|
+
return _decode(alternative, value, where)
|
|
34
|
+
except ValueError:
|
|
35
|
+
pass
|
|
36
|
+
raise ValueError(f"{where}: invalid value type")
|
|
37
|
+
if origin is tuple:
|
|
38
|
+
if not isinstance(value, list):
|
|
39
|
+
raise ValueError(f"{where}: expected an array")
|
|
40
|
+
item_type = get_args(expected)[0]
|
|
41
|
+
return tuple(_decode(item_type, item, f"{where}[{i}]") for i, item in enumerate(value))
|
|
42
|
+
if isinstance(expected, type) and is_dataclass(expected):
|
|
43
|
+
if not isinstance(value, dict):
|
|
44
|
+
raise ValueError(f"{where}: expected an object")
|
|
45
|
+
names = {field.name for field in fields(expected)}
|
|
46
|
+
if set(value) != names:
|
|
47
|
+
raise ValueError(f"{where}: missing or unexpected fields")
|
|
48
|
+
hints = get_type_hints(expected)
|
|
49
|
+
return expected(
|
|
50
|
+
**{key: _decode(hints[key], item, f"{where}.{key}") for key, item in value.items()}
|
|
51
|
+
)
|
|
52
|
+
if type(value) is not expected:
|
|
53
|
+
raise ValueError(f"{where}: expected {expected.__name__}")
|
|
54
|
+
return value
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _relative_path(value: str) -> None:
|
|
58
|
+
if (
|
|
59
|
+
not value
|
|
60
|
+
or "\\" in value
|
|
61
|
+
or ":" in value
|
|
62
|
+
or "\x00" in value
|
|
63
|
+
or PurePosixPath(value).is_absolute()
|
|
64
|
+
or ".." in PurePosixPath(value).parts
|
|
65
|
+
):
|
|
66
|
+
raise ValueError(f"Invalid project-relative path: {value!r}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _location(value: Location) -> None:
|
|
70
|
+
_relative_path(value.path)
|
|
71
|
+
if value.line < 1 or value.column < 0:
|
|
72
|
+
raise ValueError("Invalid source location")
|
|
73
|
+
if value.end_line is not None and value.end_line < value.line:
|
|
74
|
+
raise ValueError("Invalid source end line")
|
|
75
|
+
if value.end_column is not None and value.end_column < 0:
|
|
76
|
+
raise ValueError("Invalid source end column")
|
|
77
|
+
if (value.end_line is None) != (value.end_column is None):
|
|
78
|
+
raise ValueError("Source end line and column must be supplied together")
|
|
79
|
+
if value.end_line == value.line and value.end_column is not None:
|
|
80
|
+
if value.end_column < value.column:
|
|
81
|
+
raise ValueError("Source end precedes source start")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _unique_ids(items: tuple[Any, ...], kind: str) -> set[str]:
|
|
85
|
+
ids = [item.id for item in items]
|
|
86
|
+
if any(not item or "\x00" in item for item in ids) or len(set(ids)) != len(ids):
|
|
87
|
+
raise ValueError(f"Empty or duplicate {kind} IDs")
|
|
88
|
+
return set(ids)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def load_report(path: str | Path) -> LineageReport:
|
|
92
|
+
"""Read a full snapshot, rejecting malformed facts and dangling edges.
|
|
93
|
+
|
|
94
|
+
This loads plain data only. Query exports deliberately have another
|
|
95
|
+
``report_kind`` and cannot be mistaken for a complete project snapshot.
|
|
96
|
+
"""
|
|
97
|
+
try:
|
|
98
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
99
|
+
except (json.JSONDecodeError, UnicodeError) as error:
|
|
100
|
+
raise ValueError(f"Invalid JSON snapshot: {error}") from error
|
|
101
|
+
if not isinstance(data, dict):
|
|
102
|
+
raise ValueError("Snapshot must be an object")
|
|
103
|
+
if data.get("report_kind") != "column_lineage" or data.get("schema_version") not in {
|
|
104
|
+
"0.2",
|
|
105
|
+
"0.3",
|
|
106
|
+
}:
|
|
107
|
+
raise ValueError("Expected a column_lineage snapshot with schema_version 0.2 or 0.3")
|
|
108
|
+
legacy = data["schema_version"] == "0.2"
|
|
109
|
+
if legacy:
|
|
110
|
+
expected_fields = {field.name for field in fields(LineageReport)} - {"history"}
|
|
111
|
+
if set(data) != expected_fields:
|
|
112
|
+
raise ValueError("snapshot: missing or unexpected legacy fields")
|
|
113
|
+
data = {**data, "schema_version": "0.3", "history": []}
|
|
114
|
+
warnings.warn(
|
|
115
|
+
"Snapshot schema 0.2 has no recorded column history. Re-scan the project "
|
|
116
|
+
"to include sources, transformation steps, and uses.",
|
|
117
|
+
UserWarning,
|
|
118
|
+
stacklevel=2,
|
|
119
|
+
)
|
|
120
|
+
report: LineageReport = _decode(LineageReport, data, "snapshot")
|
|
121
|
+
frames = _unique_ids(report.dataframes, "DataFrame")
|
|
122
|
+
columns = _unique_ids(report.columns, "column")
|
|
123
|
+
operations = _unique_ids(report.operations, "operation")
|
|
124
|
+
history = _unique_ids(report.history, "history event")
|
|
125
|
+
if (
|
|
126
|
+
columns & operations
|
|
127
|
+
or frames & (columns | operations)
|
|
128
|
+
or history & (frames | columns | operations)
|
|
129
|
+
):
|
|
130
|
+
raise ValueError("Node IDs must be unique across record types")
|
|
131
|
+
for file in report.files:
|
|
132
|
+
_relative_path(file.path)
|
|
133
|
+
if file.sha256 is not None and (
|
|
134
|
+
len(file.sha256) != 64 or any(char not in "0123456789abcdef" for char in file.sha256)
|
|
135
|
+
):
|
|
136
|
+
raise ValueError("Invalid source sha256")
|
|
137
|
+
identities: set[tuple[str, str, int]] = set()
|
|
138
|
+
for column in report.columns:
|
|
139
|
+
if column.dataframe_id not in frames:
|
|
140
|
+
raise ValueError("Column references an unknown DataFrame")
|
|
141
|
+
if column.version < 0:
|
|
142
|
+
raise ValueError("Column version must be non-negative")
|
|
143
|
+
identity = (column.dataframe_id, column.name, column.version)
|
|
144
|
+
if identity in identities:
|
|
145
|
+
raise ValueError("Duplicate column version identity")
|
|
146
|
+
identities.add(identity)
|
|
147
|
+
for edge in report.edges:
|
|
148
|
+
if edge.kind == "read":
|
|
149
|
+
valid = edge.source in columns and edge.target in operations
|
|
150
|
+
else:
|
|
151
|
+
valid = edge.source in operations and edge.target in columns
|
|
152
|
+
if not valid:
|
|
153
|
+
raise ValueError("Invalid or dangling graph edge")
|
|
154
|
+
for consumption in report.consumptions:
|
|
155
|
+
if consumption.column_id not in columns:
|
|
156
|
+
raise ValueError("Consumption references an unknown column")
|
|
157
|
+
events = {event.id: event for event in report.history}
|
|
158
|
+
sequences: set[int] = set()
|
|
159
|
+
for event in report.history:
|
|
160
|
+
if event.sequence < 0 or event.sequence in sequences:
|
|
161
|
+
raise ValueError("History event sequence must be non-negative and unique")
|
|
162
|
+
sequences.add(event.sequence)
|
|
163
|
+
if any(frame not in frames for frame in event.dataframe_ids):
|
|
164
|
+
raise ValueError("History event references an unknown DataFrame")
|
|
165
|
+
if any(column not in columns for column in (*event.inputs, *event.outputs)):
|
|
166
|
+
raise ValueError("History event references an unknown column")
|
|
167
|
+
if any(parent not in history for parent in event.parents):
|
|
168
|
+
raise ValueError("History event references an unknown parent")
|
|
169
|
+
if any(events[parent].sequence >= event.sequence for parent in event.parents):
|
|
170
|
+
raise ValueError("History parents must precede their child events")
|
|
171
|
+
locations = (
|
|
172
|
+
[item.location for item in report.functions]
|
|
173
|
+
+ [item.location for item in report.dataframes]
|
|
174
|
+
+ [item.location for item in report.columns]
|
|
175
|
+
+ [item.location for item in report.operations]
|
|
176
|
+
+ [item.location for item in report.consumptions]
|
|
177
|
+
+ [item.location for item in report.diagnostics]
|
|
178
|
+
+ [item.location for item in report.history]
|
|
179
|
+
)
|
|
180
|
+
for location in locations:
|
|
181
|
+
_location(location)
|
|
182
|
+
return report
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _destination(path: str | Path) -> Path:
|
|
186
|
+
target = Path(path)
|
|
187
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
188
|
+
return target
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def export_json(
|
|
192
|
+
report: LineageReport,
|
|
193
|
+
path: str | Path,
|
|
194
|
+
*,
|
|
195
|
+
column_id: str | None = None,
|
|
196
|
+
direction: str = "both",
|
|
197
|
+
depth: int | None = None,
|
|
198
|
+
relation: str | None = None,
|
|
199
|
+
evidence: str = "all",
|
|
200
|
+
) -> Path:
|
|
201
|
+
"""Export a full snapshot, or an explicitly identified column query."""
|
|
202
|
+
if column_id is None:
|
|
203
|
+
if direction != "both" or depth is not None or relation is not None or evidence != "all":
|
|
204
|
+
raise ValueError("JSON query filters require a selected column")
|
|
205
|
+
content = report.to_json()
|
|
206
|
+
else:
|
|
207
|
+
query = LineageGraph(report).trace(
|
|
208
|
+
column_id, direction=direction, depth=depth, relation=relation, evidence=evidence
|
|
209
|
+
)
|
|
210
|
+
query.update(
|
|
211
|
+
report_kind="lineage_query",
|
|
212
|
+
schema_version=report.schema_version,
|
|
213
|
+
tool_version=report.tool_version,
|
|
214
|
+
)
|
|
215
|
+
content = json.dumps(query, ensure_ascii=False, sort_keys=True, indent=2)
|
|
216
|
+
target = _destination(path)
|
|
217
|
+
target.write_text(content + "\n", encoding="utf-8")
|
|
218
|
+
return target
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def export_html(
|
|
222
|
+
report: LineageReport,
|
|
223
|
+
path: str | Path,
|
|
224
|
+
*,
|
|
225
|
+
column_id: str | None = None,
|
|
226
|
+
direction: str = "both",
|
|
227
|
+
depth: int | None = None,
|
|
228
|
+
relation: str | None = None,
|
|
229
|
+
evidence: str = "all",
|
|
230
|
+
operations: bool = False,
|
|
231
|
+
view: str = "history",
|
|
232
|
+
) -> Path:
|
|
233
|
+
"""Render an offline graph. Install ``coltrace[viz]`` for this export."""
|
|
234
|
+
try:
|
|
235
|
+
from pyvis.network import Network # type: ignore[import-untyped]
|
|
236
|
+
except ImportError as error:
|
|
237
|
+
raise ValueError("HTML export requires the extra: pip install 'coltrace[viz]'") from error
|
|
238
|
+
if view not in {"history", "columns"}:
|
|
239
|
+
raise ValueError("HTML view must be 'history' or 'columns'")
|
|
240
|
+
lineage = LineageGraph(report)
|
|
241
|
+
graph = lineage._filtered(relation, evidence)
|
|
242
|
+
query: dict[str, Any] | None = None
|
|
243
|
+
if column_id is None:
|
|
244
|
+
if depth is not None or direction != "both":
|
|
245
|
+
raise ValueError("Depth and direction require a selected column")
|
|
246
|
+
included = set(lineage.by_id)
|
|
247
|
+
diagnostics = [asdict(item) for item in report.diagnostics]
|
|
248
|
+
else:
|
|
249
|
+
query = lineage.trace(
|
|
250
|
+
column_id, direction=direction, depth=depth, relation=relation, evidence=evidence
|
|
251
|
+
)
|
|
252
|
+
diagnostics = query["diagnostics"]
|
|
253
|
+
included = {column_id}
|
|
254
|
+
for side in ("upstream", "downstream"):
|
|
255
|
+
for group in ("direct", "transitive"):
|
|
256
|
+
included.update(item["id"] for item in query[side][group])
|
|
257
|
+
projected = [
|
|
258
|
+
(source, target, data)
|
|
259
|
+
for source, target, data in graph.edges(data=True)
|
|
260
|
+
if source in included and target in included
|
|
261
|
+
]
|
|
262
|
+
net = Network(height="100%", width="100%", directed=True, cdn_resources="in_line")
|
|
263
|
+
history = query.get("history", {}) if query is not None else {}
|
|
264
|
+
journey = query.get("journey", {}) if query is not None else {}
|
|
265
|
+
history_mode = bool(history.get("recorded") and view == "history" and not operations)
|
|
266
|
+
history_notice = ""
|
|
267
|
+
if column_id is not None and view == "history" and not operations and not history_mode:
|
|
268
|
+
history_notice = (
|
|
269
|
+
"This snapshot has no recorded column history. Re-scan the project to include "
|
|
270
|
+
"source expressions and transformation steps. Showing identified dependencies."
|
|
271
|
+
)
|
|
272
|
+
stages: list[dict[str, Any]] = []
|
|
273
|
+
if history_mode:
|
|
274
|
+
primary = {**history, "stages": journey.get("steps", history.get("stages", []))}
|
|
275
|
+
stages = _history_nodes(net, primary, column_id)
|
|
276
|
+
options = {
|
|
277
|
+
"interaction": {
|
|
278
|
+
"hover": True,
|
|
279
|
+
"keyboard": {"enabled": True, "bindToWindow": False},
|
|
280
|
+
"zoomSpeed": 0.7,
|
|
281
|
+
},
|
|
282
|
+
"layout": {"hierarchical": {"enabled": False}},
|
|
283
|
+
"physics": {"enabled": False},
|
|
284
|
+
"edges": {
|
|
285
|
+
"smooth": {"enabled": True, "type": "cubicBezier", "forceDirection": "vertical"},
|
|
286
|
+
"color": {"color": "#94a3b8", "highlight": "#0f766e"},
|
|
287
|
+
"width": 1.6,
|
|
288
|
+
},
|
|
289
|
+
}
|
|
290
|
+
return _write_html(
|
|
291
|
+
net,
|
|
292
|
+
report,
|
|
293
|
+
path,
|
|
294
|
+
diagnostics,
|
|
295
|
+
column_id,
|
|
296
|
+
options,
|
|
297
|
+
history_mode=True,
|
|
298
|
+
stages=stages,
|
|
299
|
+
history_notice="",
|
|
300
|
+
journey=journey,
|
|
301
|
+
creation_stages=history.get("stages", []),
|
|
302
|
+
)
|
|
303
|
+
for column in report.columns:
|
|
304
|
+
if column.id not in included:
|
|
305
|
+
continue
|
|
306
|
+
details = json.dumps(asdict(column), ensure_ascii=False, indent=2)
|
|
307
|
+
net.add_node(
|
|
308
|
+
column.id,
|
|
309
|
+
label=f"{column.name} · v{column.version}",
|
|
310
|
+
shape="box",
|
|
311
|
+
title=html.escape(f"{column.location.path}:{column.location.line} · {column.scope}"),
|
|
312
|
+
details=details,
|
|
313
|
+
color="#e2e8f0",
|
|
314
|
+
selected=column.id == column_id,
|
|
315
|
+
)
|
|
316
|
+
if operations:
|
|
317
|
+
operation_ids = {data["operation_id"] for _, _, data in projected}
|
|
318
|
+
visible_edges = set()
|
|
319
|
+
for source, target, data in projected:
|
|
320
|
+
visible_edges.add((source, data["operation_id"], "read"))
|
|
321
|
+
visible_edges.add((data["operation_id"], target, "write"))
|
|
322
|
+
by_operation = {operation.id: operation for operation in report.operations}
|
|
323
|
+
operations_with_reads = {edge.target for edge in report.edges if edge.kind == "read"}
|
|
324
|
+
# Only actual input-free writes lack a projection. Including all writes
|
|
325
|
+
# here would reintroduce relations excluded by confidence filters.
|
|
326
|
+
for edge in report.edges:
|
|
327
|
+
if edge.kind != "write" or edge.target not in included:
|
|
328
|
+
continue
|
|
329
|
+
if edge.source in operations_with_reads:
|
|
330
|
+
continue
|
|
331
|
+
operation = by_operation[edge.source]
|
|
332
|
+
kind = operation.kind if operation.kind in {"rename", "pass"} else "derive"
|
|
333
|
+
confirmed = all(
|
|
334
|
+
value == "confirmed"
|
|
335
|
+
for value in (
|
|
336
|
+
edge.evidence,
|
|
337
|
+
operation.evidence,
|
|
338
|
+
lineage.by_id[edge.target].evidence,
|
|
339
|
+
)
|
|
340
|
+
)
|
|
341
|
+
if (relation is None or relation == kind) and (evidence == "all" or confirmed):
|
|
342
|
+
operation_ids.add(operation.id)
|
|
343
|
+
visible_edges.add((edge.source, edge.target, edge.kind))
|
|
344
|
+
for operation in report.operations:
|
|
345
|
+
if operation.id in operation_ids:
|
|
346
|
+
net.add_node(
|
|
347
|
+
operation.id,
|
|
348
|
+
label=operation.kind,
|
|
349
|
+
shape="diamond",
|
|
350
|
+
color="#cbd5e1",
|
|
351
|
+
title=html.escape(
|
|
352
|
+
f"{operation.location.path}:{operation.location.line} · {operation.scope}"
|
|
353
|
+
),
|
|
354
|
+
details=json.dumps(asdict(operation), ensure_ascii=False, indent=2),
|
|
355
|
+
)
|
|
356
|
+
for edge in report.edges:
|
|
357
|
+
if (edge.source, edge.target, edge.kind) in visible_edges:
|
|
358
|
+
operation_id = edge.target if edge.kind == "read" else edge.source
|
|
359
|
+
endpoint = edge.source if edge.kind == "read" else edge.target
|
|
360
|
+
confirmed = all(
|
|
361
|
+
value == "confirmed"
|
|
362
|
+
for value in (
|
|
363
|
+
edge.evidence,
|
|
364
|
+
by_operation[operation_id].evidence,
|
|
365
|
+
lineage.by_id[endpoint].evidence,
|
|
366
|
+
)
|
|
367
|
+
)
|
|
368
|
+
net.add_edge(
|
|
369
|
+
edge.source,
|
|
370
|
+
edge.target,
|
|
371
|
+
label=edge.kind,
|
|
372
|
+
dashes=not confirmed,
|
|
373
|
+
arrows="to",
|
|
374
|
+
)
|
|
375
|
+
else:
|
|
376
|
+
for source, target, data in projected:
|
|
377
|
+
net.add_edge(
|
|
378
|
+
source,
|
|
379
|
+
target,
|
|
380
|
+
label=data["kind"],
|
|
381
|
+
arrows="to",
|
|
382
|
+
title=html.escape(
|
|
383
|
+
f"{data['location']['path']}:{data['location']['line']} · "
|
|
384
|
+
f"{data['evidence']} · {data['completeness']}"
|
|
385
|
+
),
|
|
386
|
+
dashes=data["evidence"] != "confirmed",
|
|
387
|
+
)
|
|
388
|
+
options = {
|
|
389
|
+
"interaction": {
|
|
390
|
+
"hover": True,
|
|
391
|
+
"keyboard": {"enabled": True, "bindToWindow": False},
|
|
392
|
+
"zoomSpeed": 0.7,
|
|
393
|
+
},
|
|
394
|
+
"layout": {"hierarchical": {"enabled": False}},
|
|
395
|
+
"physics": {"enabled": False},
|
|
396
|
+
"edges": {"smooth": {"enabled": True, "type": "cubicBezier", "forceDirection": "vertical"}},
|
|
397
|
+
}
|
|
398
|
+
return _write_html(
|
|
399
|
+
net,
|
|
400
|
+
report,
|
|
401
|
+
path,
|
|
402
|
+
diagnostics,
|
|
403
|
+
column_id,
|
|
404
|
+
options,
|
|
405
|
+
history_mode=False,
|
|
406
|
+
stages=stages,
|
|
407
|
+
history_notice=history_notice,
|
|
408
|
+
journey=journey,
|
|
409
|
+
creation_stages=[],
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _callback_title(stage: dict[str, Any]) -> str:
|
|
414
|
+
"""Keep argument-heavy expressions in the sidebar rather than the canvas."""
|
|
415
|
+
label = str(stage["label"])
|
|
416
|
+
if label.startswith("Type guard;"):
|
|
417
|
+
return "Validate input type"
|
|
418
|
+
if label.startswith("Return component"):
|
|
419
|
+
return f"{stage['label']}: {stage['expression']}"
|
|
420
|
+
target = label.partition(" = ")[0]
|
|
421
|
+
try:
|
|
422
|
+
expression = str(stage["expression"])
|
|
423
|
+
if len(expression) > 4096:
|
|
424
|
+
return target
|
|
425
|
+
root = ast.parse(expression, mode="eval")
|
|
426
|
+
except (SyntaxError, RecursionError):
|
|
427
|
+
return target
|
|
428
|
+
calls: list[str] = []
|
|
429
|
+
|
|
430
|
+
def visit(node: ast.AST) -> None:
|
|
431
|
+
for child in ast.iter_child_nodes(node):
|
|
432
|
+
visit(child)
|
|
433
|
+
if isinstance(node, ast.Call):
|
|
434
|
+
function = node.func
|
|
435
|
+
if isinstance(function, ast.Name):
|
|
436
|
+
calls.append(function.id)
|
|
437
|
+
elif isinstance(function, ast.Attribute):
|
|
438
|
+
owner = f"{function.value.id}." if isinstance(function.value, ast.Name) else ""
|
|
439
|
+
calls.append(owner + function.attr)
|
|
440
|
+
|
|
441
|
+
try:
|
|
442
|
+
visit(root)
|
|
443
|
+
except RecursionError:
|
|
444
|
+
return target
|
|
445
|
+
summary = " → ".join(dict.fromkeys(calls))
|
|
446
|
+
return f"{target}\n{summary}" if summary else label
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _stage_title(stage: dict[str, Any], outputs: list[str]) -> str:
|
|
450
|
+
kind = stage["kind"]
|
|
451
|
+
label = str(stage["label"])
|
|
452
|
+
if stage.get("continuity") == "unresolved":
|
|
453
|
+
return "Later export\nColumn survival unverified"
|
|
454
|
+
if kind.startswith("callback_"):
|
|
455
|
+
return _callback_title(stage)
|
|
456
|
+
if kind == "source":
|
|
457
|
+
return label.removeprefix("Create DataFrame with ")
|
|
458
|
+
if kind == "external":
|
|
459
|
+
return "Input: " + ", ".join(outputs)
|
|
460
|
+
if kind == "bind":
|
|
461
|
+
return "Bind DataFrame: " + label.partition(" <- ")[0]
|
|
462
|
+
if kind == "call":
|
|
463
|
+
function = label.rsplit(".", 1)[-1]
|
|
464
|
+
return (
|
|
465
|
+
"Function template\n" if label.startswith("Analyze function") else "Call: "
|
|
466
|
+
) + function
|
|
467
|
+
if kind == "return":
|
|
468
|
+
return "Return DataFrame" if stage["dataframe_ids"] else "Return values"
|
|
469
|
+
if kind in {"filter", "dropna", "projection"}:
|
|
470
|
+
if stage.get("relationship") == "projection_exclusion":
|
|
471
|
+
return "Exclude column from\nresulting DataFrame"
|
|
472
|
+
if stage.get("relationship") == "direct_read":
|
|
473
|
+
return "Read column\nto select rows"
|
|
474
|
+
if stage.get("relationship") == "frame_pass":
|
|
475
|
+
return "Carry column into\nresulting DataFrame"
|
|
476
|
+
return "Select columns" if kind == "projection" else "Filter rows\nPreserve retained values"
|
|
477
|
+
if kind in {"derive", "apply", "overwrite", "rename", "source"} and outputs:
|
|
478
|
+
verb = {"rename": "Rename", "overwrite": "Overwrite"}.get(kind, "Write")
|
|
479
|
+
return verb + ": " + ", ".join(outputs)
|
|
480
|
+
if kind == "boundary":
|
|
481
|
+
return (
|
|
482
|
+
"Unresolved origin: " + ", ".join(outputs)
|
|
483
|
+
if outputs
|
|
484
|
+
else "Lineage gap\nUnknown call or mutation"
|
|
485
|
+
)
|
|
486
|
+
if kind == "sink":
|
|
487
|
+
return "Export DataFrame"
|
|
488
|
+
return label
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _history_nodes(
|
|
492
|
+
net: Any, history: dict[str, Any], column_id: str | None
|
|
493
|
+
) -> list[dict[str, Any]]:
|
|
494
|
+
"""Show recorded steps; raw column versions remain in the JSON facts."""
|
|
495
|
+
nodes = history.get("nodes", [])
|
|
496
|
+
names = {
|
|
497
|
+
node["id"]: f"{node['name']} · v{node['version']}"
|
|
498
|
+
for node in nodes
|
|
499
|
+
if node.get("node_type") == "column"
|
|
500
|
+
}
|
|
501
|
+
plain_names = {node["id"]: node["name"] for node in nodes if node.get("node_type") == "column"}
|
|
502
|
+
palette = {
|
|
503
|
+
"source": {"background": "#eff6ff", "border": "#90b3df"},
|
|
504
|
+
"transform": {"background": "#ffffff", "border": "#b8afd5"},
|
|
505
|
+
"use": {"background": "#eef8f4", "border": "#8cbfab"},
|
|
506
|
+
"boundary": {"background": "#fff8e8", "border": "#d7ae57"},
|
|
507
|
+
"frame": {"background": "#f0f6f9", "border": "#9fb5c5"},
|
|
508
|
+
"derivation": {"background": "#f5efff", "border": "#aa8bca"},
|
|
509
|
+
"control": {"background": "#f3f5f7", "border": "#a7b3bf"},
|
|
510
|
+
"configuration": {"background": "#f7f4ef", "border": "#beaf96"},
|
|
511
|
+
"context": {"background": "#f3f5f7", "border": "#a7b3bf"},
|
|
512
|
+
}
|
|
513
|
+
stages: list[dict[str, Any]] = []
|
|
514
|
+
remap: dict[str, str] = {}
|
|
515
|
+
for stage in history.get("stages", []):
|
|
516
|
+
for event_id in stage.get("event_ids", [stage["id"]]):
|
|
517
|
+
remap[event_id] = stage["id"]
|
|
518
|
+
for output in stage["outputs"]:
|
|
519
|
+
remap[output] = stage["id"]
|
|
520
|
+
inputs = list(dict.fromkeys(names[key] for key in stage["inputs"] if key in names))
|
|
521
|
+
outputs = list(dict.fromkeys(names[key] for key in stage["outputs"] if key in names))
|
|
522
|
+
flow = ""
|
|
523
|
+
if inputs or outputs:
|
|
524
|
+
flow = f"Columns: {', '.join(inputs) or '(source)'} → {', '.join(outputs) or '(use)'}"
|
|
525
|
+
stages.append({**stage, "column_flow": flow})
|
|
526
|
+
location = stage["location"]
|
|
527
|
+
category = stage["category"]
|
|
528
|
+
title = _stage_title(
|
|
529
|
+
stage, [plain_names[key] for key in stage["outputs"] if key in plain_names]
|
|
530
|
+
)
|
|
531
|
+
title_lines = [
|
|
532
|
+
line for part in title.splitlines() for line in textwrap.wrap(part, width=30)
|
|
533
|
+
]
|
|
534
|
+
if len(title_lines) > 3:
|
|
535
|
+
title_lines = title_lines[:3]
|
|
536
|
+
title_lines[-1] = title_lines[-1][:27] + "..."
|
|
537
|
+
number = len(stages)
|
|
538
|
+
role = str(stage.get("relationship", category)).replace("_", " ")
|
|
539
|
+
label = f"{number:02d} · {role.upper()}\n" + "\n".join(title_lines)
|
|
540
|
+
label += f"\n{location['path'].rsplit('/', 1)[-1]}:{location['line']}"
|
|
541
|
+
details = (
|
|
542
|
+
f"{stage['label']}\n{location['path']}:{location['line']}\n"
|
|
543
|
+
f"{stage['module']}::{stage['scope']}\n\n{stage['expression']}\n\n{flow}\n"
|
|
544
|
+
f"{stage.get('explanation', '')}\n"
|
|
545
|
+
f"Evidence: {stage['evidence']}\nCompleteness: {stage['completeness']}"
|
|
546
|
+
)
|
|
547
|
+
color = (
|
|
548
|
+
palette["boundary"]
|
|
549
|
+
if stage.get("continuity") == "unresolved"
|
|
550
|
+
else palette.get(category, palette["context"])
|
|
551
|
+
)
|
|
552
|
+
net.add_node(
|
|
553
|
+
stage["id"],
|
|
554
|
+
label=label,
|
|
555
|
+
shape="box",
|
|
556
|
+
color={
|
|
557
|
+
**color,
|
|
558
|
+
"highlight": {"background": color["background"], "border": "#087f8c"},
|
|
559
|
+
"hover": {"background": color["background"], "border": "#087f8c"},
|
|
560
|
+
},
|
|
561
|
+
details=details,
|
|
562
|
+
category=category,
|
|
563
|
+
stage_number=number,
|
|
564
|
+
widthConstraint={"minimum": 280, "maximum": 280},
|
|
565
|
+
heightConstraint={"minimum": 110},
|
|
566
|
+
margin={"top": 20, "right": 22, "bottom": 20, "left": 22},
|
|
567
|
+
font={"size": 16, "face": "Segoe UI", "color": "#26354b", "align": "left"},
|
|
568
|
+
borderWidth=1.5,
|
|
569
|
+
borderWidthSelected=3,
|
|
570
|
+
shapeProperties={
|
|
571
|
+
"borderRadius": 8,
|
|
572
|
+
"borderDashes": category == "boundary" or stage.get("continuity") == "unresolved",
|
|
573
|
+
},
|
|
574
|
+
title=html.escape(f"{location['path']}:{location['line']} · {stage['scope']}"),
|
|
575
|
+
selected=column_id in stage["outputs"],
|
|
576
|
+
)
|
|
577
|
+
graph: nx.DiGraph[str] = nx.DiGraph()
|
|
578
|
+
visible_ids = {stage["id"] for stage in stages}
|
|
579
|
+
isolated = {stage["id"] for stage in stages if stage.get("continuity") == "unresolved"}
|
|
580
|
+
graph.add_nodes_from(visible_ids)
|
|
581
|
+
recorded: nx.DiGraph[str] = nx.DiGraph()
|
|
582
|
+
for edge in history.get("edges", []):
|
|
583
|
+
source = remap.get(edge["source"], edge["source"])
|
|
584
|
+
target = remap.get(edge["target"], edge["target"])
|
|
585
|
+
if source != target and source not in isolated and target not in isolated:
|
|
586
|
+
recorded.add_edge(source, target, **edge)
|
|
587
|
+
# Hidden callback preparation stays available in creation context. Compress
|
|
588
|
+
# its recorded causal paths; never connect blocks merely by source order.
|
|
589
|
+
for source in sorted(visible_ids - isolated):
|
|
590
|
+
pending = [(source, "confirmed", "step")]
|
|
591
|
+
visited = {source}
|
|
592
|
+
confidence_rank = {"confirmed": 0, "inferred": 1, "unresolved": 2}
|
|
593
|
+
while pending:
|
|
594
|
+
current, confidence, kind = pending.pop()
|
|
595
|
+
if current not in recorded:
|
|
596
|
+
continue
|
|
597
|
+
for target, attributes in recorded[current].items():
|
|
598
|
+
evidence = attributes.get("evidence", "confirmed")
|
|
599
|
+
combined = max(
|
|
600
|
+
(confidence, evidence), key=lambda value: confidence_rank.get(value, 2)
|
|
601
|
+
)
|
|
602
|
+
edge_kind = attributes.get("kind", kind)
|
|
603
|
+
if target in visible_ids:
|
|
604
|
+
if target != source:
|
|
605
|
+
graph.add_edge(source, target, evidence=combined, kind=edge_kind)
|
|
606
|
+
elif target not in visited:
|
|
607
|
+
visited.add(target)
|
|
608
|
+
pending.append((target, combined, edge_kind))
|
|
609
|
+
# Remove visual shortcuts already explained by recorded step chains. No
|
|
610
|
+
# causal edge is inferred merely from the order of statements or lines.
|
|
611
|
+
visible = nx.transitive_reduction(graph) if nx.is_directed_acyclic_graph(graph) else graph
|
|
612
|
+
for source, target in visible.edges:
|
|
613
|
+
edge = graph[source][target]
|
|
614
|
+
net.add_edge(
|
|
615
|
+
source,
|
|
616
|
+
target,
|
|
617
|
+
arrows="to",
|
|
618
|
+
dashes=edge.get("evidence") != "confirmed" or edge.get("kind") == "supersedes",
|
|
619
|
+
title=html.escape(str(edge.get("kind", ""))),
|
|
620
|
+
label="supersedes (history only)" if edge.get("kind") == "supersedes" else "",
|
|
621
|
+
)
|
|
622
|
+
return stages
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _write_html(
|
|
626
|
+
net: Any,
|
|
627
|
+
report: LineageReport,
|
|
628
|
+
path: str | Path,
|
|
629
|
+
diagnostics: list[dict[str, Any]],
|
|
630
|
+
column_id: str | None,
|
|
631
|
+
options: dict[str, Any],
|
|
632
|
+
*,
|
|
633
|
+
history_mode: bool,
|
|
634
|
+
stages: list[dict[str, Any]],
|
|
635
|
+
history_notice: str,
|
|
636
|
+
journey: dict[str, Any],
|
|
637
|
+
creation_stages: list[dict[str, Any]],
|
|
638
|
+
) -> Path:
|
|
639
|
+
selected = next((column for column in report.columns if column.id == column_id), None)
|
|
640
|
+
selected_label = f"{selected.name} · {selected.module}::{selected.scope}" if selected else ""
|
|
641
|
+
positions = layout_history(net.nodes, net.edges)
|
|
642
|
+
for node in net.nodes:
|
|
643
|
+
node.update(positions[node["id"]])
|
|
644
|
+
net.set_options(json.dumps(options))
|
|
645
|
+
net.templateEnv.globals.update(
|
|
646
|
+
diagnostics=diagnostics,
|
|
647
|
+
filtered=column_id is not None,
|
|
648
|
+
history_mode=history_mode,
|
|
649
|
+
stages=stages,
|
|
650
|
+
journey=journey,
|
|
651
|
+
creation_stages=creation_stages,
|
|
652
|
+
history_notice=history_notice,
|
|
653
|
+
selected_label=selected_label,
|
|
654
|
+
selected_name=selected.name if selected else "Project dependencies",
|
|
655
|
+
selected_scope=f"{selected.module}::{selected.scope}"
|
|
656
|
+
if selected
|
|
657
|
+
else "All identified columns",
|
|
658
|
+
tool_version=report.tool_version,
|
|
659
|
+
column_notice=(
|
|
660
|
+
"Column provenance is unresolved. DataFrame source metadata is context, "
|
|
661
|
+
"not proof of this column's origin."
|
|
662
|
+
)
|
|
663
|
+
if selected is not None
|
|
664
|
+
and (
|
|
665
|
+
selected.origin == "opaque"
|
|
666
|
+
or selected.evidence == "unresolved"
|
|
667
|
+
or selected.completeness == "unresolved"
|
|
668
|
+
)
|
|
669
|
+
else "",
|
|
670
|
+
)
|
|
671
|
+
template = files("coltrace").joinpath("templates/graph.html").read_text(encoding="utf-8")
|
|
672
|
+
net.template = net.templateEnv.from_string(template)
|
|
673
|
+
content: str = net.generate_html(notebook=True)
|
|
674
|
+
output_path = _destination(path)
|
|
675
|
+
output_path.write_text(content, encoding="utf-8")
|
|
676
|
+
return output_path
|