tracepipe 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

tracepipe/value_provenance.py

@@ -0,0 +1,301 @@
+# tracepipe/value_provenance.py
+"""
+Cell-level value provenance tracking.
+
+Provides detailed history of how specific cell values changed
+throughout the pipeline, including null introduction tracking.
+
+Usage:
+    # Get history of a specific cell
+    history = tp.explain_value(row_id=123, column="price", df=result)
+
+    # Analyze where nulls came from in a column
+    analysis = tp.null_analysis("email", df)
+"""
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import pandas as pd
+
+from .context import get_context
+from .core import ChangeType
+
+
+@dataclass
+class ValueEvent:
+    """Single change event for a cell."""
+
+    step_id: int
+    operation: str
+    old_value: Any
+    new_value: Any
+    change_type: str
+    timestamp: float
+    code_location: Optional[str]
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "step_id": self.step_id,
+            "operation": self.operation,
+            "old_value": self.old_value,
+            "new_value": self.new_value,
+            "change_type": self.change_type,
+            "timestamp": self.timestamp,
+            "code_location": self.code_location,
+        }
+
+
+@dataclass
+class ValueHistory:
+    """Complete history of a cell's value."""
+
+    row_id: int
+    column: str
+    current_value: Any
+    events: list[ValueEvent]
+    became_null_at: Optional[int] = None  # step_id
+    became_null_by: Optional[str] = None  # operation
+
+    def __repr__(self) -> str:
+        lines = [f"Value History: row {self.row_id}, column '{self.column}'"]
+        lines.append(f"  Current: {self.current_value}")
+        lines.append(f"  Changes: {len(self.events)}")
+
+        if self.became_null_at:
+            lines.append(f"  ! Became null at step {self.became_null_at} by {self.became_null_by}")
+
+        for event in self.events[-5:]:
+            lines.append(f"    {event.operation}: {event.old_value} -> {event.new_value}")
+
+        if len(self.events) > 5:
+            lines.append(f"    ... and {len(self.events) - 5} more events")
+
+        return "\n".join(lines)
+
+    @property
+    def was_modified(self) -> bool:
+        """True if value was ever modified."""
+        return len(self.events) > 0
+
+    @property
+    def is_null(self) -> bool:
+        """True if current value is null."""
+        return pd.isna(self.current_value)
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "row_id": self.row_id,
+            "column": self.column,
+            "current_value": self.current_value,
+            "events": [e.to_dict() for e in self.events],
+            "became_null_at": self.became_null_at,
+            "became_null_by": self.became_null_by,
+        }
+
+
+def explain_value(row_id: int, column: str, df: Optional[pd.DataFrame] = None) -> ValueHistory:
+    """
+    Get complete history of a specific cell's value.
+
+    Args:
+        row_id: Row ID to trace
+        column: Column name
+        df: Optional DataFrame for current value lookup
+
+    Returns:
+        ValueHistory with all changes to this cell
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    # Get current value if df provided
+    current_value = None
+    if df is not None:
+        rids = ctx.row_manager.get_ids_array(df)
+        if rids is not None:
+            # Find position of this row_id
+            matches = (rids == row_id).nonzero()[0]
+            if len(matches) > 0 and column in df.columns:
+                current_value = df.iloc[matches[0]][column]
+
+    # Collect all events for this cell
+    events = []
+    step_map = {s.step_id: s for s in store.steps}
+    became_null_at = None
+    became_null_by = None
+
+    for diff in store._iter_all_diffs():
+        if diff["row_id"] == row_id and diff["col"] == column:
+            step = step_map.get(diff["step_id"])
+
+            events.append(
+                ValueEvent(
+                    step_id=diff["step_id"],
+                    operation=step.operation if step else "unknown",
+                    old_value=diff["old_val"],
+                    new_value=diff["new_val"],
+                    change_type=ChangeType(diff["change_type"]).name,
+                    timestamp=step.timestamp if step else 0,
+                    code_location=(
+                        f"{step.code_file}:{step.code_line}" if step and step.code_file else None
+                    ),
+                )
+            )
+
+            # Track when value became null
+            if became_null_at is None and pd.isna(diff["new_val"]) and not pd.isna(diff["old_val"]):
+                became_null_at = diff["step_id"]
+                became_null_by = step.operation if step else "unknown"
+
+    events.sort(key=lambda e: e.step_id)
+
+    return ValueHistory(
+        row_id=row_id,
+        column=column,
+        current_value=current_value,
+        events=events,
+        became_null_at=became_null_at,
+        became_null_by=became_null_by,
+    )
+
+
+@dataclass
+class NullAnalysis:
+    """Analysis of how nulls appeared in a column."""
+
+    column: str
+    total_nulls: int
+    null_sources: dict[str, int]  # operation -> count
+    sample_row_ids: list[int]
+
+    def __repr__(self) -> str:
+        lines = [f"Null Analysis: '{self.column}'"]
+        lines.append(f"  Total nulls: {self.total_nulls}")
+
+        if self.null_sources:
+            lines.append("  Sources:")
+            for op, count in sorted(self.null_sources.items(), key=lambda x: -x[1]):
+                lines.append(f"    {op}: {count}")
+        else:
+            lines.append("  No tracked null introductions")
+
+        if self.sample_row_ids:
+            lines.append(f"  Sample row IDs: {self.sample_row_ids[:5]}")
+
+        return "\n".join(lines)
+
+    @property
+    def has_untracked_nulls(self) -> bool:
+        """True if some nulls were not tracked by TracePipe."""
+        tracked = sum(self.null_sources.values())
+        return tracked < self.total_nulls
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "column": self.column,
+            "total_nulls": self.total_nulls,
+            "null_sources": self.null_sources,
+            "sample_row_ids": self.sample_row_ids,
+            "has_untracked_nulls": self.has_untracked_nulls,
+        }
+
+
+def null_analysis(column: str, df: pd.DataFrame) -> NullAnalysis:
+    """
+    Analyze how nulls appeared in a column.
+
+    Returns breakdown of which operations introduced nulls.
+
+    Args:
+        column: Column name to analyze
+        df: Current DataFrame
+
+    Returns:
+        NullAnalysis with breakdown of null sources
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    if column not in df.columns:
+        return NullAnalysis(column=column, total_nulls=0, null_sources={}, sample_row_ids=[])
+
+    rids = ctx.row_manager.get_ids_array(df)
+    if rids is None:
+        return NullAnalysis(
+            column=column,
+            total_nulls=int(df[column].isna().sum()),
+            null_sources={},
+            sample_row_ids=[],
+        )
+
+    # Find null rows
+    null_mask = df[column].isna()
+    null_rids = set(rids[null_mask].tolist())
+
+    # Track which operations introduced nulls
+    null_sources: dict[str, int] = {}
+    step_map = {s.step_id: s for s in store.steps}
+    sample_ids: list[int] = []
+
+    for diff in store._iter_all_diffs():
+        if diff["col"] == column and diff["row_id"] in null_rids:
+            if pd.isna(diff["new_val"]) and not pd.isna(diff["old_val"]):
+                step = step_map.get(diff["step_id"])
+                op = step.operation if step else "unknown"
+                null_sources[op] = null_sources.get(op, 0) + 1
+                if len(sample_ids) < 10:
+                    sample_ids.append(diff["row_id"])
+
+    return NullAnalysis(
+        column=column,
+        total_nulls=len(null_rids),
+        null_sources=null_sources,
+        sample_row_ids=sample_ids,
+    )
+
+
+def column_changes_summary(column: str, df: pd.DataFrame) -> dict[str, Any]:
+    """
+    Get summary of all changes to a column.
+
+    Args:
+        column: Column name
+        df: Current DataFrame
+
+    Returns:
+        Dict with summary statistics
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    rids = ctx.row_manager.get_ids_array(df)
+    if rids is None:
+        return {
+            "column": column,
+            "total_changes": 0,
+            "changes_by_operation": {},
+            "unique_rows_modified": 0,
+        }
+
+    rid_set = set(rids.tolist())
+    changes_by_op: dict[str, int] = {}
+    modified_rows: set = set()
+    step_map = {s.step_id: s for s in store.steps}
+
+    for diff in store._iter_all_diffs():
+        if diff["col"] == column and diff["row_id"] in rid_set:
+            step = step_map.get(diff["step_id"])
+            op = step.operation if step else "unknown"
+            changes_by_op[op] = changes_by_op.get(op, 0) + 1
+            modified_rows.add(diff["row_id"])
+
+    return {
+        "column": column,
+        "total_changes": sum(changes_by_op.values()),
+        "changes_by_operation": changes_by_op,
+        "unique_rows_modified": len(modified_rows),
+    }

tracepipe/visualization/html_export.py

@@ -156,12 +156,17 @@ def _get_groups_summary(ctx) -> list[dict]:
     groups = []
     for mapping in ctx.store.aggregation_mappings:
         for group_key, row_ids in mapping.membership.items():
-            is_count_only = isinstance(row_ids, int)
+            # Count-only groups are stored as [-count] (list with one negative element)
+            is_count_only = len(row_ids) == 1 and row_ids[0] < 0
+            if is_count_only:
+                row_count = abs(row_ids[0])
+            else:
+                row_count = len(row_ids)
             groups.append(
                 {
                     "key": str(group_key),
                     "column": mapping.group_column,
-                    "row_count": row_ids if is_count_only else len(row_ids),
+                    "row_count": row_count,
                     "is_count_only": is_count_only,
                     "row_ids": [] if is_count_only else row_ids[:100],  # First 100 only
                     "agg_functions": mapping.agg_functions,
@@ -1059,9 +1064,13 @@ document.addEventListener('DOMContentLoaded', () => {
 """
 
 
-def save(filepath: str) -> None:
+def save(filepath: str, title: str = "TracePipe Dashboard") -> None:
     """
     Save interactive lineage report as HTML.
+
+    Args:
+        filepath: Path to save the HTML file
+        title: Title for the report (shown in browser tab and header)
     """
     ctx = get_context()
 
@@ -1073,7 +1082,9 @@ def save(filepath: str) -> None:
     row_index = _build_row_index(ctx)
 
     # Total registered rows (approximate)
-    total_registered = ctx.row_manager.next_row_id if hasattr(ctx.row_manager, "next_row_id") else 0
+    total_registered = (
+        ctx.row_manager._next_row_id if hasattr(ctx.row_manager, "_next_row_id") else 0
+    )
 
     # Identify Suggested Rows for UX
     suggested_rows = {"dropped": [], "modified": [], "survivors": []}
@@ -1181,13 +1192,16 @@ def save(filepath: str) -> None:
         </div>
         """
 
+    # Escape title for HTML
+    escaped_title = html.escape(title)
+
     html_content = f"""
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>TracePipe Dashboard</title>
+    <title>{escaped_title}</title>
     {CSS}
 </head>
 <body>
@@ -1217,7 +1231,7 @@ def save(filepath: str) -> None:
         <div class="main-content">
             <!-- Top Bar -->
             <div class="top-bar">
-                <div class="page-title">Data Lineage Report</div>
+                <div class="page-title">{escaped_title}</div>
                 <div class="search-wrapper">
                     <i class="search-icon-abs">🔍</i>
                     <input type="text" id="globalSearch" class="search-input"
@@ -1236,7 +1250,8 @@ def save(filepath: str) -> None:
                     </div>
                     <div class="card">
                         <h3>Retention</h3>
-                        <div class="metric-value">{(final_rows / initial_rows * 100) if initial_rows else 0:.1f}%</div>
+                        <div class="metric-value">{
+        (final_rows / initial_rows * 100) if initial_rows else 0:.1f}%</div>
                         <div class="metric-sub">{_format_number(final_rows)} of {
         _format_number(initial_rows)
     } rows</div>