coltrace 0.1.0.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
coltrace/exporters.py ADDED
@@ -0,0 +1,676 @@
1
+ """Validated JSON snapshots and self-contained, offline graph exports."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ast
6
+ import html
7
+ import json
8
+ import textwrap
9
+ import types
10
+ import warnings
11
+ from dataclasses import asdict, fields, is_dataclass
12
+ from importlib.resources import files
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import Any, Literal, get_args, get_origin, get_type_hints
15
+
16
+ import networkx as nx
17
+
18
+ from .graph import LineageGraph
19
+ from .lineage_models import LineageReport
20
+ from .models import Location
21
+ from .visual_layout import layout_history
22
+
23
+
24
+ def _decode(expected: Any, value: Any, where: str) -> Any:
25
+ origin = get_origin(expected)
26
+ if origin is Literal:
27
+ if value not in get_args(expected) or not isinstance(value, str):
28
+ raise ValueError(f"{where}: invalid enum value")
29
+ return value
30
+ if origin is types.UnionType:
31
+ for alternative in get_args(expected):
32
+ try:
33
+ return _decode(alternative, value, where)
34
+ except ValueError:
35
+ pass
36
+ raise ValueError(f"{where}: invalid value type")
37
+ if origin is tuple:
38
+ if not isinstance(value, list):
39
+ raise ValueError(f"{where}: expected an array")
40
+ item_type = get_args(expected)[0]
41
+ return tuple(_decode(item_type, item, f"{where}[{i}]") for i, item in enumerate(value))
42
+ if isinstance(expected, type) and is_dataclass(expected):
43
+ if not isinstance(value, dict):
44
+ raise ValueError(f"{where}: expected an object")
45
+ names = {field.name for field in fields(expected)}
46
+ if set(value) != names:
47
+ raise ValueError(f"{where}: missing or unexpected fields")
48
+ hints = get_type_hints(expected)
49
+ return expected(
50
+ **{key: _decode(hints[key], item, f"{where}.{key}") for key, item in value.items()}
51
+ )
52
+ if type(value) is not expected:
53
+ raise ValueError(f"{where}: expected {expected.__name__}")
54
+ return value
55
+
56
+
57
+ def _relative_path(value: str) -> None:
58
+ if (
59
+ not value
60
+ or "\\" in value
61
+ or ":" in value
62
+ or "\x00" in value
63
+ or PurePosixPath(value).is_absolute()
64
+ or ".." in PurePosixPath(value).parts
65
+ ):
66
+ raise ValueError(f"Invalid project-relative path: {value!r}")
67
+
68
+
69
+ def _location(value: Location) -> None:
70
+ _relative_path(value.path)
71
+ if value.line < 1 or value.column < 0:
72
+ raise ValueError("Invalid source location")
73
+ if value.end_line is not None and value.end_line < value.line:
74
+ raise ValueError("Invalid source end line")
75
+ if value.end_column is not None and value.end_column < 0:
76
+ raise ValueError("Invalid source end column")
77
+ if (value.end_line is None) != (value.end_column is None):
78
+ raise ValueError("Source end line and column must be supplied together")
79
+ if value.end_line == value.line and value.end_column is not None:
80
+ if value.end_column < value.column:
81
+ raise ValueError("Source end precedes source start")
82
+
83
+
84
+ def _unique_ids(items: tuple[Any, ...], kind: str) -> set[str]:
85
+ ids = [item.id for item in items]
86
+ if any(not item or "\x00" in item for item in ids) or len(set(ids)) != len(ids):
87
+ raise ValueError(f"Empty or duplicate {kind} IDs")
88
+ return set(ids)
89
+
90
+
91
+ def load_report(path: str | Path) -> LineageReport:
92
+ """Read a full snapshot, rejecting malformed facts and dangling edges.
93
+
94
+ This loads plain data only. Query exports deliberately have another
95
+ ``report_kind`` and cannot be mistaken for a complete project snapshot.
96
+ """
97
+ try:
98
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
99
+ except (json.JSONDecodeError, UnicodeError) as error:
100
+ raise ValueError(f"Invalid JSON snapshot: {error}") from error
101
+ if not isinstance(data, dict):
102
+ raise ValueError("Snapshot must be an object")
103
+ if data.get("report_kind") != "column_lineage" or data.get("schema_version") not in {
104
+ "0.2",
105
+ "0.3",
106
+ }:
107
+ raise ValueError("Expected a column_lineage snapshot with schema_version 0.2 or 0.3")
108
+ legacy = data["schema_version"] == "0.2"
109
+ if legacy:
110
+ expected_fields = {field.name for field in fields(LineageReport)} - {"history"}
111
+ if set(data) != expected_fields:
112
+ raise ValueError("snapshot: missing or unexpected legacy fields")
113
+ data = {**data, "schema_version": "0.3", "history": []}
114
+ warnings.warn(
115
+ "Snapshot schema 0.2 has no recorded column history. Re-scan the project "
116
+ "to include sources, transformation steps, and uses.",
117
+ UserWarning,
118
+ stacklevel=2,
119
+ )
120
+ report: LineageReport = _decode(LineageReport, data, "snapshot")
121
+ frames = _unique_ids(report.dataframes, "DataFrame")
122
+ columns = _unique_ids(report.columns, "column")
123
+ operations = _unique_ids(report.operations, "operation")
124
+ history = _unique_ids(report.history, "history event")
125
+ if (
126
+ columns & operations
127
+ or frames & (columns | operations)
128
+ or history & (frames | columns | operations)
129
+ ):
130
+ raise ValueError("Node IDs must be unique across record types")
131
+ for file in report.files:
132
+ _relative_path(file.path)
133
+ if file.sha256 is not None and (
134
+ len(file.sha256) != 64 or any(char not in "0123456789abcdef" for char in file.sha256)
135
+ ):
136
+ raise ValueError("Invalid source sha256")
137
+ identities: set[tuple[str, str, int]] = set()
138
+ for column in report.columns:
139
+ if column.dataframe_id not in frames:
140
+ raise ValueError("Column references an unknown DataFrame")
141
+ if column.version < 0:
142
+ raise ValueError("Column version must be non-negative")
143
+ identity = (column.dataframe_id, column.name, column.version)
144
+ if identity in identities:
145
+ raise ValueError("Duplicate column version identity")
146
+ identities.add(identity)
147
+ for edge in report.edges:
148
+ if edge.kind == "read":
149
+ valid = edge.source in columns and edge.target in operations
150
+ else:
151
+ valid = edge.source in operations and edge.target in columns
152
+ if not valid:
153
+ raise ValueError("Invalid or dangling graph edge")
154
+ for consumption in report.consumptions:
155
+ if consumption.column_id not in columns:
156
+ raise ValueError("Consumption references an unknown column")
157
+ events = {event.id: event for event in report.history}
158
+ sequences: set[int] = set()
159
+ for event in report.history:
160
+ if event.sequence < 0 or event.sequence in sequences:
161
+ raise ValueError("History event sequence must be non-negative and unique")
162
+ sequences.add(event.sequence)
163
+ if any(frame not in frames for frame in event.dataframe_ids):
164
+ raise ValueError("History event references an unknown DataFrame")
165
+ if any(column not in columns for column in (*event.inputs, *event.outputs)):
166
+ raise ValueError("History event references an unknown column")
167
+ if any(parent not in history for parent in event.parents):
168
+ raise ValueError("History event references an unknown parent")
169
+ if any(events[parent].sequence >= event.sequence for parent in event.parents):
170
+ raise ValueError("History parents must precede their child events")
171
+ locations = (
172
+ [item.location for item in report.functions]
173
+ + [item.location for item in report.dataframes]
174
+ + [item.location for item in report.columns]
175
+ + [item.location for item in report.operations]
176
+ + [item.location for item in report.consumptions]
177
+ + [item.location for item in report.diagnostics]
178
+ + [item.location for item in report.history]
179
+ )
180
+ for location in locations:
181
+ _location(location)
182
+ return report
183
+
184
+
185
+ def _destination(path: str | Path) -> Path:
186
+ target = Path(path)
187
+ target.parent.mkdir(parents=True, exist_ok=True)
188
+ return target
189
+
190
+
191
+ def export_json(
192
+ report: LineageReport,
193
+ path: str | Path,
194
+ *,
195
+ column_id: str | None = None,
196
+ direction: str = "both",
197
+ depth: int | None = None,
198
+ relation: str | None = None,
199
+ evidence: str = "all",
200
+ ) -> Path:
201
+ """Export a full snapshot, or an explicitly identified column query."""
202
+ if column_id is None:
203
+ if direction != "both" or depth is not None or relation is not None or evidence != "all":
204
+ raise ValueError("JSON query filters require a selected column")
205
+ content = report.to_json()
206
+ else:
207
+ query = LineageGraph(report).trace(
208
+ column_id, direction=direction, depth=depth, relation=relation, evidence=evidence
209
+ )
210
+ query.update(
211
+ report_kind="lineage_query",
212
+ schema_version=report.schema_version,
213
+ tool_version=report.tool_version,
214
+ )
215
+ content = json.dumps(query, ensure_ascii=False, sort_keys=True, indent=2)
216
+ target = _destination(path)
217
+ target.write_text(content + "\n", encoding="utf-8")
218
+ return target
219
+
220
+
221
+ def export_html(
222
+ report: LineageReport,
223
+ path: str | Path,
224
+ *,
225
+ column_id: str | None = None,
226
+ direction: str = "both",
227
+ depth: int | None = None,
228
+ relation: str | None = None,
229
+ evidence: str = "all",
230
+ operations: bool = False,
231
+ view: str = "history",
232
+ ) -> Path:
233
+ """Render an offline graph. Install ``coltrace[viz]`` for this export."""
234
+ try:
235
+ from pyvis.network import Network # type: ignore[import-untyped]
236
+ except ImportError as error:
237
+ raise ValueError("HTML export requires the extra: pip install 'coltrace[viz]'") from error
238
+ if view not in {"history", "columns"}:
239
+ raise ValueError("HTML view must be 'history' or 'columns'")
240
+ lineage = LineageGraph(report)
241
+ graph = lineage._filtered(relation, evidence)
242
+ query: dict[str, Any] | None = None
243
+ if column_id is None:
244
+ if depth is not None or direction != "both":
245
+ raise ValueError("Depth and direction require a selected column")
246
+ included = set(lineage.by_id)
247
+ diagnostics = [asdict(item) for item in report.diagnostics]
248
+ else:
249
+ query = lineage.trace(
250
+ column_id, direction=direction, depth=depth, relation=relation, evidence=evidence
251
+ )
252
+ diagnostics = query["diagnostics"]
253
+ included = {column_id}
254
+ for side in ("upstream", "downstream"):
255
+ for group in ("direct", "transitive"):
256
+ included.update(item["id"] for item in query[side][group])
257
+ projected = [
258
+ (source, target, data)
259
+ for source, target, data in graph.edges(data=True)
260
+ if source in included and target in included
261
+ ]
262
+ net = Network(height="100%", width="100%", directed=True, cdn_resources="in_line")
263
+ history = query.get("history", {}) if query is not None else {}
264
+ journey = query.get("journey", {}) if query is not None else {}
265
+ history_mode = bool(history.get("recorded") and view == "history" and not operations)
266
+ history_notice = ""
267
+ if column_id is not None and view == "history" and not operations and not history_mode:
268
+ history_notice = (
269
+ "This snapshot has no recorded column history. Re-scan the project to include "
270
+ "source expressions and transformation steps. Showing identified dependencies."
271
+ )
272
+ stages: list[dict[str, Any]] = []
273
+ if history_mode:
274
+ primary = {**history, "stages": journey.get("steps", history.get("stages", []))}
275
+ stages = _history_nodes(net, primary, column_id)
276
+ options = {
277
+ "interaction": {
278
+ "hover": True,
279
+ "keyboard": {"enabled": True, "bindToWindow": False},
280
+ "zoomSpeed": 0.7,
281
+ },
282
+ "layout": {"hierarchical": {"enabled": False}},
283
+ "physics": {"enabled": False},
284
+ "edges": {
285
+ "smooth": {"enabled": True, "type": "cubicBezier", "forceDirection": "vertical"},
286
+ "color": {"color": "#94a3b8", "highlight": "#0f766e"},
287
+ "width": 1.6,
288
+ },
289
+ }
290
+ return _write_html(
291
+ net,
292
+ report,
293
+ path,
294
+ diagnostics,
295
+ column_id,
296
+ options,
297
+ history_mode=True,
298
+ stages=stages,
299
+ history_notice="",
300
+ journey=journey,
301
+ creation_stages=history.get("stages", []),
302
+ )
303
+ for column in report.columns:
304
+ if column.id not in included:
305
+ continue
306
+ details = json.dumps(asdict(column), ensure_ascii=False, indent=2)
307
+ net.add_node(
308
+ column.id,
309
+ label=f"{column.name} · v{column.version}",
310
+ shape="box",
311
+ title=html.escape(f"{column.location.path}:{column.location.line} · {column.scope}"),
312
+ details=details,
313
+ color="#e2e8f0",
314
+ selected=column.id == column_id,
315
+ )
316
+ if operations:
317
+ operation_ids = {data["operation_id"] for _, _, data in projected}
318
+ visible_edges = set()
319
+ for source, target, data in projected:
320
+ visible_edges.add((source, data["operation_id"], "read"))
321
+ visible_edges.add((data["operation_id"], target, "write"))
322
+ by_operation = {operation.id: operation for operation in report.operations}
323
+ operations_with_reads = {edge.target for edge in report.edges if edge.kind == "read"}
324
+ # Only actual input-free writes lack a projection. Including all writes
325
+ # here would reintroduce relations excluded by confidence filters.
326
+ for edge in report.edges:
327
+ if edge.kind != "write" or edge.target not in included:
328
+ continue
329
+ if edge.source in operations_with_reads:
330
+ continue
331
+ operation = by_operation[edge.source]
332
+ kind = operation.kind if operation.kind in {"rename", "pass"} else "derive"
333
+ confirmed = all(
334
+ value == "confirmed"
335
+ for value in (
336
+ edge.evidence,
337
+ operation.evidence,
338
+ lineage.by_id[edge.target].evidence,
339
+ )
340
+ )
341
+ if (relation is None or relation == kind) and (evidence == "all" or confirmed):
342
+ operation_ids.add(operation.id)
343
+ visible_edges.add((edge.source, edge.target, edge.kind))
344
+ for operation in report.operations:
345
+ if operation.id in operation_ids:
346
+ net.add_node(
347
+ operation.id,
348
+ label=operation.kind,
349
+ shape="diamond",
350
+ color="#cbd5e1",
351
+ title=html.escape(
352
+ f"{operation.location.path}:{operation.location.line} · {operation.scope}"
353
+ ),
354
+ details=json.dumps(asdict(operation), ensure_ascii=False, indent=2),
355
+ )
356
+ for edge in report.edges:
357
+ if (edge.source, edge.target, edge.kind) in visible_edges:
358
+ operation_id = edge.target if edge.kind == "read" else edge.source
359
+ endpoint = edge.source if edge.kind == "read" else edge.target
360
+ confirmed = all(
361
+ value == "confirmed"
362
+ for value in (
363
+ edge.evidence,
364
+ by_operation[operation_id].evidence,
365
+ lineage.by_id[endpoint].evidence,
366
+ )
367
+ )
368
+ net.add_edge(
369
+ edge.source,
370
+ edge.target,
371
+ label=edge.kind,
372
+ dashes=not confirmed,
373
+ arrows="to",
374
+ )
375
+ else:
376
+ for source, target, data in projected:
377
+ net.add_edge(
378
+ source,
379
+ target,
380
+ label=data["kind"],
381
+ arrows="to",
382
+ title=html.escape(
383
+ f"{data['location']['path']}:{data['location']['line']} · "
384
+ f"{data['evidence']} · {data['completeness']}"
385
+ ),
386
+ dashes=data["evidence"] != "confirmed",
387
+ )
388
+ options = {
389
+ "interaction": {
390
+ "hover": True,
391
+ "keyboard": {"enabled": True, "bindToWindow": False},
392
+ "zoomSpeed": 0.7,
393
+ },
394
+ "layout": {"hierarchical": {"enabled": False}},
395
+ "physics": {"enabled": False},
396
+ "edges": {"smooth": {"enabled": True, "type": "cubicBezier", "forceDirection": "vertical"}},
397
+ }
398
+ return _write_html(
399
+ net,
400
+ report,
401
+ path,
402
+ diagnostics,
403
+ column_id,
404
+ options,
405
+ history_mode=False,
406
+ stages=stages,
407
+ history_notice=history_notice,
408
+ journey=journey,
409
+ creation_stages=[],
410
+ )
411
+
412
+
413
+ def _callback_title(stage: dict[str, Any]) -> str:
414
+ """Keep argument-heavy expressions in the sidebar rather than the canvas."""
415
+ label = str(stage["label"])
416
+ if label.startswith("Type guard;"):
417
+ return "Validate input type"
418
+ if label.startswith("Return component"):
419
+ return f"{stage['label']}: {stage['expression']}"
420
+ target = label.partition(" = ")[0]
421
+ try:
422
+ expression = str(stage["expression"])
423
+ if len(expression) > 4096:
424
+ return target
425
+ root = ast.parse(expression, mode="eval")
426
+ except (SyntaxError, RecursionError):
427
+ return target
428
+ calls: list[str] = []
429
+
430
+ def visit(node: ast.AST) -> None:
431
+ for child in ast.iter_child_nodes(node):
432
+ visit(child)
433
+ if isinstance(node, ast.Call):
434
+ function = node.func
435
+ if isinstance(function, ast.Name):
436
+ calls.append(function.id)
437
+ elif isinstance(function, ast.Attribute):
438
+ owner = f"{function.value.id}." if isinstance(function.value, ast.Name) else ""
439
+ calls.append(owner + function.attr)
440
+
441
+ try:
442
+ visit(root)
443
+ except RecursionError:
444
+ return target
445
+ summary = " → ".join(dict.fromkeys(calls))
446
+ return f"{target}\n{summary}" if summary else label
447
+
448
+
449
+ def _stage_title(stage: dict[str, Any], outputs: list[str]) -> str:
450
+ kind = stage["kind"]
451
+ label = str(stage["label"])
452
+ if stage.get("continuity") == "unresolved":
453
+ return "Later export\nColumn survival unverified"
454
+ if kind.startswith("callback_"):
455
+ return _callback_title(stage)
456
+ if kind == "source":
457
+ return label.removeprefix("Create DataFrame with ")
458
+ if kind == "external":
459
+ return "Input: " + ", ".join(outputs)
460
+ if kind == "bind":
461
+ return "Bind DataFrame: " + label.partition(" <- ")[0]
462
+ if kind == "call":
463
+ function = label.rsplit(".", 1)[-1]
464
+ return (
465
+ "Function template\n" if label.startswith("Analyze function") else "Call: "
466
+ ) + function
467
+ if kind == "return":
468
+ return "Return DataFrame" if stage["dataframe_ids"] else "Return values"
469
+ if kind in {"filter", "dropna", "projection"}:
470
+ if stage.get("relationship") == "projection_exclusion":
471
+ return "Exclude column from\nresulting DataFrame"
472
+ if stage.get("relationship") == "direct_read":
473
+ return "Read column\nto select rows"
474
+ if stage.get("relationship") == "frame_pass":
475
+ return "Carry column into\nresulting DataFrame"
476
+ return "Select columns" if kind == "projection" else "Filter rows\nPreserve retained values"
477
+ if kind in {"derive", "apply", "overwrite", "rename", "source"} and outputs:
478
+ verb = {"rename": "Rename", "overwrite": "Overwrite"}.get(kind, "Write")
479
+ return verb + ": " + ", ".join(outputs)
480
+ if kind == "boundary":
481
+ return (
482
+ "Unresolved origin: " + ", ".join(outputs)
483
+ if outputs
484
+ else "Lineage gap\nUnknown call or mutation"
485
+ )
486
+ if kind == "sink":
487
+ return "Export DataFrame"
488
+ return label
489
+
490
+
491
+ def _history_nodes(
492
+ net: Any, history: dict[str, Any], column_id: str | None
493
+ ) -> list[dict[str, Any]]:
494
+ """Show recorded steps; raw column versions remain in the JSON facts."""
495
+ nodes = history.get("nodes", [])
496
+ names = {
497
+ node["id"]: f"{node['name']} · v{node['version']}"
498
+ for node in nodes
499
+ if node.get("node_type") == "column"
500
+ }
501
+ plain_names = {node["id"]: node["name"] for node in nodes if node.get("node_type") == "column"}
502
+ palette = {
503
+ "source": {"background": "#eff6ff", "border": "#90b3df"},
504
+ "transform": {"background": "#ffffff", "border": "#b8afd5"},
505
+ "use": {"background": "#eef8f4", "border": "#8cbfab"},
506
+ "boundary": {"background": "#fff8e8", "border": "#d7ae57"},
507
+ "frame": {"background": "#f0f6f9", "border": "#9fb5c5"},
508
+ "derivation": {"background": "#f5efff", "border": "#aa8bca"},
509
+ "control": {"background": "#f3f5f7", "border": "#a7b3bf"},
510
+ "configuration": {"background": "#f7f4ef", "border": "#beaf96"},
511
+ "context": {"background": "#f3f5f7", "border": "#a7b3bf"},
512
+ }
513
+ stages: list[dict[str, Any]] = []
514
+ remap: dict[str, str] = {}
515
+ for stage in history.get("stages", []):
516
+ for event_id in stage.get("event_ids", [stage["id"]]):
517
+ remap[event_id] = stage["id"]
518
+ for output in stage["outputs"]:
519
+ remap[output] = stage["id"]
520
+ inputs = list(dict.fromkeys(names[key] for key in stage["inputs"] if key in names))
521
+ outputs = list(dict.fromkeys(names[key] for key in stage["outputs"] if key in names))
522
+ flow = ""
523
+ if inputs or outputs:
524
+ flow = f"Columns: {', '.join(inputs) or '(source)'} → {', '.join(outputs) or '(use)'}"
525
+ stages.append({**stage, "column_flow": flow})
526
+ location = stage["location"]
527
+ category = stage["category"]
528
+ title = _stage_title(
529
+ stage, [plain_names[key] for key in stage["outputs"] if key in plain_names]
530
+ )
531
+ title_lines = [
532
+ line for part in title.splitlines() for line in textwrap.wrap(part, width=30)
533
+ ]
534
+ if len(title_lines) > 3:
535
+ title_lines = title_lines[:3]
536
+ title_lines[-1] = title_lines[-1][:27] + "..."
537
+ number = len(stages)
538
+ role = str(stage.get("relationship", category)).replace("_", " ")
539
+ label = f"{number:02d} · {role.upper()}\n" + "\n".join(title_lines)
540
+ label += f"\n{location['path'].rsplit('/', 1)[-1]}:{location['line']}"
541
+ details = (
542
+ f"{stage['label']}\n{location['path']}:{location['line']}\n"
543
+ f"{stage['module']}::{stage['scope']}\n\n{stage['expression']}\n\n{flow}\n"
544
+ f"{stage.get('explanation', '')}\n"
545
+ f"Evidence: {stage['evidence']}\nCompleteness: {stage['completeness']}"
546
+ )
547
+ color = (
548
+ palette["boundary"]
549
+ if stage.get("continuity") == "unresolved"
550
+ else palette.get(category, palette["context"])
551
+ )
552
+ net.add_node(
553
+ stage["id"],
554
+ label=label,
555
+ shape="box",
556
+ color={
557
+ **color,
558
+ "highlight": {"background": color["background"], "border": "#087f8c"},
559
+ "hover": {"background": color["background"], "border": "#087f8c"},
560
+ },
561
+ details=details,
562
+ category=category,
563
+ stage_number=number,
564
+ widthConstraint={"minimum": 280, "maximum": 280},
565
+ heightConstraint={"minimum": 110},
566
+ margin={"top": 20, "right": 22, "bottom": 20, "left": 22},
567
+ font={"size": 16, "face": "Segoe UI", "color": "#26354b", "align": "left"},
568
+ borderWidth=1.5,
569
+ borderWidthSelected=3,
570
+ shapeProperties={
571
+ "borderRadius": 8,
572
+ "borderDashes": category == "boundary" or stage.get("continuity") == "unresolved",
573
+ },
574
+ title=html.escape(f"{location['path']}:{location['line']} · {stage['scope']}"),
575
+ selected=column_id in stage["outputs"],
576
+ )
577
+ graph: nx.DiGraph[str] = nx.DiGraph()
578
+ visible_ids = {stage["id"] for stage in stages}
579
+ isolated = {stage["id"] for stage in stages if stage.get("continuity") == "unresolved"}
580
+ graph.add_nodes_from(visible_ids)
581
+ recorded: nx.DiGraph[str] = nx.DiGraph()
582
+ for edge in history.get("edges", []):
583
+ source = remap.get(edge["source"], edge["source"])
584
+ target = remap.get(edge["target"], edge["target"])
585
+ if source != target and source not in isolated and target not in isolated:
586
+ recorded.add_edge(source, target, **edge)
587
+ # Hidden callback preparation stays available in creation context. Compress
588
+ # its recorded causal paths; never connect blocks merely by source order.
589
+ for source in sorted(visible_ids - isolated):
590
+ pending = [(source, "confirmed", "step")]
591
+ visited = {source}
592
+ confidence_rank = {"confirmed": 0, "inferred": 1, "unresolved": 2}
593
+ while pending:
594
+ current, confidence, kind = pending.pop()
595
+ if current not in recorded:
596
+ continue
597
+ for target, attributes in recorded[current].items():
598
+ evidence = attributes.get("evidence", "confirmed")
599
+ combined = max(
600
+ (confidence, evidence), key=lambda value: confidence_rank.get(value, 2)
601
+ )
602
+ edge_kind = attributes.get("kind", kind)
603
+ if target in visible_ids:
604
+ if target != source:
605
+ graph.add_edge(source, target, evidence=combined, kind=edge_kind)
606
+ elif target not in visited:
607
+ visited.add(target)
608
+ pending.append((target, combined, edge_kind))
609
+ # Remove visual shortcuts already explained by recorded step chains. No
610
+ # causal edge is inferred merely from the order of statements or lines.
611
+ visible = nx.transitive_reduction(graph) if nx.is_directed_acyclic_graph(graph) else graph
612
+ for source, target in visible.edges:
613
+ edge = graph[source][target]
614
+ net.add_edge(
615
+ source,
616
+ target,
617
+ arrows="to",
618
+ dashes=edge.get("evidence") != "confirmed" or edge.get("kind") == "supersedes",
619
+ title=html.escape(str(edge.get("kind", ""))),
620
+ label="supersedes (history only)" if edge.get("kind") == "supersedes" else "",
621
+ )
622
+ return stages
623
+
624
+
625
+ def _write_html(
626
+ net: Any,
627
+ report: LineageReport,
628
+ path: str | Path,
629
+ diagnostics: list[dict[str, Any]],
630
+ column_id: str | None,
631
+ options: dict[str, Any],
632
+ *,
633
+ history_mode: bool,
634
+ stages: list[dict[str, Any]],
635
+ history_notice: str,
636
+ journey: dict[str, Any],
637
+ creation_stages: list[dict[str, Any]],
638
+ ) -> Path:
639
+ selected = next((column for column in report.columns if column.id == column_id), None)
640
+ selected_label = f"{selected.name} · {selected.module}::{selected.scope}" if selected else ""
641
+ positions = layout_history(net.nodes, net.edges)
642
+ for node in net.nodes:
643
+ node.update(positions[node["id"]])
644
+ net.set_options(json.dumps(options))
645
+ net.templateEnv.globals.update(
646
+ diagnostics=diagnostics,
647
+ filtered=column_id is not None,
648
+ history_mode=history_mode,
649
+ stages=stages,
650
+ journey=journey,
651
+ creation_stages=creation_stages,
652
+ history_notice=history_notice,
653
+ selected_label=selected_label,
654
+ selected_name=selected.name if selected else "Project dependencies",
655
+ selected_scope=f"{selected.module}::{selected.scope}"
656
+ if selected
657
+ else "All identified columns",
658
+ tool_version=report.tool_version,
659
+ column_notice=(
660
+ "Column provenance is unresolved. DataFrame source metadata is context, "
661
+ "not proof of this column's origin."
662
+ )
663
+ if selected is not None
664
+ and (
665
+ selected.origin == "opaque"
666
+ or selected.evidence == "unresolved"
667
+ or selected.completeness == "unresolved"
668
+ )
669
+ else "",
670
+ )
671
+ template = files("coltrace").joinpath("templates/graph.html").read_text(encoding="utf-8")
672
+ net.template = net.templateEnv.from_string(template)
673
+ content: str = net.generate_html(notebook=True)
674
+ output_path = _destination(path)
675
+ output_path.write_text(content, encoding="utf-8")
676
+ return output_path