tracepipe 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

tracepipe/value_provenance.py

@@ -0,0 +1,309 @@
+# tracepipe/value_provenance.py
+"""
+Cell-level value provenance tracking.
+
+Provides detailed history of how specific cell values changed
+throughout the pipeline, including null introduction tracking.
+
+Usage:
+    # Get history of a specific cell
+    history = tp.explain_value(row_id=123, column="price", df=result)
+
+    # Analyze where nulls came from in a column
+    analysis = tp.null_analysis("email", df)
+"""
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import pandas as pd
+
+from .context import get_context
+
+
+@dataclass
+class ValueEvent:
+    """Single change event for a cell."""
+
+    step_id: int
+    operation: str
+    old_value: Any
+    new_value: Any
+    change_type: str
+    timestamp: float
+    code_location: Optional[str]
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "step_id": self.step_id,
+            "operation": self.operation,
+            "old_value": self.old_value,
+            "new_value": self.new_value,
+            "change_type": self.change_type,
+            "timestamp": self.timestamp,
+            "code_location": self.code_location,
+        }
+
+
+@dataclass
+class ValueHistory:
+    """Complete history of a cell's value."""
+
+    row_id: int
+    column: str
+    current_value: Any
+    events: list[ValueEvent]
+    became_null_at: Optional[int] = None  # step_id
+    became_null_by: Optional[str] = None  # operation
+
+    def __repr__(self) -> str:
+        lines = [f"Value History: row {self.row_id}, column '{self.column}'"]
+        lines.append(f"  Current: {self.current_value}")
+        lines.append(f"  Changes: {len(self.events)}")
+
+        if self.became_null_at:
+            lines.append(f"  ! Became null at step {self.became_null_at} by {self.became_null_by}")
+
+        for event in self.events[-5:]:
+            lines.append(f"    {event.operation}: {event.old_value} -> {event.new_value}")
+
+        if len(self.events) > 5:
+            lines.append(f"    ... and {len(self.events) - 5} more events")
+
+        return "\n".join(lines)
+
+    @property
+    def was_modified(self) -> bool:
+        """True if value was ever modified."""
+        return len(self.events) > 0
+
+    @property
+    def is_null(self) -> bool:
+        """True if current value is null."""
+        return pd.isna(self.current_value)
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "row_id": self.row_id,
+            "column": self.column,
+            "current_value": self.current_value,
+            "events": [e.to_dict() for e in self.events],
+            "became_null_at": self.became_null_at,
+            "became_null_by": self.became_null_by,
+        }
+
+
+def explain_value(
+    row_id: int,
+    column: str,
+    df: Optional[pd.DataFrame] = None,
+    follow_lineage: bool = True,
+) -> ValueHistory:
+    """
+    Get complete history of a specific cell's value.
+
+    Args:
+        row_id: Row ID to trace
+        column: Column name
+        df: Optional DataFrame for current value lookup
+        follow_lineage: If True, include pre-merge parent history (default: True)
+
+    Returns:
+        ValueHistory with all changes to this cell
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    # Get current value if df provided
+    current_value = None
+    if df is not None:
+        rids = ctx.row_manager.get_ids_array(df)
+        if rids is not None:
+            # Find position of this row_id
+            matches = (rids == row_id).nonzero()[0]
+            if len(matches) > 0 and column in df.columns:
+                current_value = df.iloc[matches[0]][column]
+
+    # Collect events - use lineage-aware method if requested
+    if follow_lineage and hasattr(store, "get_cell_history_with_lineage"):
+        # Get cell history including pre-merge parent history
+        raw_events = store.get_cell_history_with_lineage(row_id, column)
+    else:
+        # Fallback to direct row_id lookup only
+        raw_events = [e for e in store.get_row_history(row_id) if e["col"] == column]
+
+    # Convert to ValueEvent objects
+    events = []
+    became_null_at = None
+    became_null_by = None
+
+    for diff in raw_events:
+        events.append(
+            ValueEvent(
+                step_id=diff["step_id"],
+                operation=diff.get("operation", "unknown"),
+                old_value=diff["old_val"],
+                new_value=diff["new_val"],
+                change_type=diff.get("change_type", "UNKNOWN"),
+                timestamp=diff.get("timestamp", 0) or 0,
+                code_location=diff.get("code_location"),
+            )
+        )
+
+        # Track when value became null
+        if became_null_at is None and pd.isna(diff["new_val"]) and not pd.isna(diff["old_val"]):
+            became_null_at = diff["step_id"]
+            became_null_by = diff.get("operation", "unknown")
+
+    # Events should already be sorted by step_id from lineage method
+    events.sort(key=lambda e: e.step_id)
+
+    return ValueHistory(
+        row_id=row_id,
+        column=column,
+        current_value=current_value,
+        events=events,
+        became_null_at=became_null_at,
+        became_null_by=became_null_by,
+    )
+
+
+@dataclass
+class NullAnalysis:
+    """Analysis of how nulls appeared in a column."""
+
+    column: str
+    total_nulls: int
+    null_sources: dict[str, int]  # operation -> count
+    sample_row_ids: list[int]
+
+    def __repr__(self) -> str:
+        lines = [f"Null Analysis: '{self.column}'"]
+        lines.append(f"  Total nulls: {self.total_nulls}")
+
+        if self.null_sources:
+            lines.append("  Sources:")
+            for op, count in sorted(self.null_sources.items(), key=lambda x: -x[1]):
+                lines.append(f"    {op}: {count}")
+        else:
+            lines.append("  No tracked null introductions")
+
+        if self.sample_row_ids:
+            lines.append(f"  Sample row IDs: {self.sample_row_ids[:5]}")
+
+        return "\n".join(lines)
+
+    @property
+    def has_untracked_nulls(self) -> bool:
+        """True if some nulls were not tracked by TracePipe."""
+        tracked = sum(self.null_sources.values())
+        return tracked < self.total_nulls
+
+    def to_dict(self) -> dict:
+        """Export to dictionary."""
+        return {
+            "column": self.column,
+            "total_nulls": self.total_nulls,
+            "null_sources": self.null_sources,
+            "sample_row_ids": self.sample_row_ids,
+            "has_untracked_nulls": self.has_untracked_nulls,
+        }
+
+
+def null_analysis(column: str, df: pd.DataFrame) -> NullAnalysis:
+    """
+    Analyze how nulls appeared in a column.
+
+    Returns breakdown of which operations introduced nulls.
+
+    Args:
+        column: Column name to analyze
+        df: Current DataFrame
+
+    Returns:
+        NullAnalysis with breakdown of null sources
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    if column not in df.columns:
+        return NullAnalysis(column=column, total_nulls=0, null_sources={}, sample_row_ids=[])
+
+    rids = ctx.row_manager.get_ids_array(df)
+    if rids is None:
+        return NullAnalysis(
+            column=column,
+            total_nulls=int(df[column].isna().sum()),
+            null_sources={},
+            sample_row_ids=[],
+        )
+
+    # Find null rows
+    null_mask = df[column].isna()
+    null_rids = set(rids[null_mask].tolist())
+
+    # Track which operations introduced nulls
+    null_sources: dict[str, int] = {}
+    step_map = {s.step_id: s for s in store.steps}
+    sample_ids: list[int] = []
+
+    for diff in store._iter_all_diffs():
+        if diff["col"] == column and diff["row_id"] in null_rids:
+            if pd.isna(diff["new_val"]) and not pd.isna(diff["old_val"]):
+                step = step_map.get(diff["step_id"])
+                op = step.operation if step else "unknown"
+                null_sources[op] = null_sources.get(op, 0) + 1
+                if len(sample_ids) < 10:
+                    sample_ids.append(diff["row_id"])
+
+    return NullAnalysis(
+        column=column,
+        total_nulls=len(null_rids),
+        null_sources=null_sources,
+        sample_row_ids=sample_ids,
+    )
+
+
+def column_changes_summary(column: str, df: pd.DataFrame) -> dict[str, Any]:
+    """
+    Get summary of all changes to a column.
+
+    Args:
+        column: Column name
+        df: Current DataFrame
+
+    Returns:
+        Dict with summary statistics
+    """
+    ctx = get_context()
+    store = ctx.store
+
+    rids = ctx.row_manager.get_ids_array(df)
+    if rids is None:
+        return {
+            "column": column,
+            "total_changes": 0,
+            "changes_by_operation": {},
+            "unique_rows_modified": 0,
+        }
+
+    rid_set = set(rids.tolist())
+    changes_by_op: dict[str, int] = {}
+    modified_rows: set = set()
+    step_map = {s.step_id: s for s in store.steps}
+
+    for diff in store._iter_all_diffs():
+        if diff["col"] == column and diff["row_id"] in rid_set:
+            step = step_map.get(diff["step_id"])
+            op = step.operation if step else "unknown"
+            changes_by_op[op] = changes_by_op.get(op, 0) + 1
+            modified_rows.add(diff["row_id"])
+
+    return {
+        "column": column,
+        "total_changes": sum(changes_by_op.values()),
+        "changes_by_operation": changes_by_op,
+        "unique_rows_modified": len(modified_rows),
+    }

tracepipe/visualization/html_export.py

@@ -156,12 +156,17 @@ def _get_groups_summary(ctx) -> list[dict]:
     groups = []
     for mapping in ctx.store.aggregation_mappings:
         for group_key, row_ids in mapping.membership.items():
-            is_count_only = isinstance(row_ids, int)
+            # Count-only groups are stored as [-count] (list with one negative element)
+            is_count_only = len(row_ids) == 1 and row_ids[0] < 0
+            if is_count_only:
+                row_count = abs(row_ids[0])
+            else:
+                row_count = len(row_ids)
             groups.append(
                 {
                     "key": str(group_key),
                     "column": mapping.group_column,
-                    "row_count": row_ids if is_count_only else len(row_ids),
+                    "row_count": row_count,
                     "is_count_only": is_count_only,
                     "row_ids": [] if is_count_only else row_ids[:100],  # First 100 only
                     "agg_functions": mapping.agg_functions,
@@ -1059,9 +1064,13 @@ document.addEventListener('DOMContentLoaded', () => {
 """
 
 
-def save(filepath: str) -> None:
+def save(filepath: str, title: str = "TracePipe Dashboard") -> None:
     """
     Save interactive lineage report as HTML.
+
+    Args:
+        filepath: Path to save the HTML file
+        title: Title for the report (shown in browser tab and header)
     """
     ctx = get_context()
 
@@ -1073,7 +1082,9 @@ def save(filepath: str) -> None:
     row_index = _build_row_index(ctx)
 
     # Total registered rows (approximate)
-    total_registered = ctx.row_manager.next_row_id if hasattr(ctx.row_manager, "next_row_id") else 0
+    total_registered = (
+        ctx.row_manager._next_row_id if hasattr(ctx.row_manager, "_next_row_id") else 0
+    )
 
     # Identify Suggested Rows for UX
     suggested_rows = {"dropped": [], "modified": [], "survivors": []}
@@ -1181,13 +1192,16 @@ def save(filepath: str) -> None:
         </div>
         """
 
+    # Escape title for HTML
+    escaped_title = html.escape(title)
+
     html_content = f"""
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>TracePipe Dashboard</title>
+    <title>{escaped_title}</title>
     {CSS}
 </head>
 <body>
@@ -1217,7 +1231,7 @@ def save(filepath: str) -> None:
         <div class="main-content">
             <!-- Top Bar -->
             <div class="top-bar">
-                <div class="page-title">Data Lineage Report</div>
+                <div class="page-title">{escaped_title}</div>
                 <div class="search-wrapper">
                     <i class="search-icon-abs">🔍</i>
                     <input type="text" id="globalSearch" class="search-input"
@@ -1236,7 +1250,8 @@ def save(filepath: str) -> None:
                     </div>
                     <div class="card">
                         <h3>Retention</h3>
-                        <div class="metric-value">{(final_rows / initial_rows * 100) if initial_rows else 0:.1f}%</div>
+                        <div class="metric-value">{
+        (final_rows / initial_rows * 100) if initial_rows else 0:.1f}%</div>
                         <div class="metric-sub">{_format_number(final_rows)} of {
         _format_number(initial_rows)
     } rows</div>

tracepipe-0.3.1.dist-info/METADATA

@@ -0,0 +1,308 @@
+Metadata-Version: 2.4
+Name: tracepipe
+Version: 0.3.1
+Summary: Row-level data lineage tracking for pandas pipelines
+Project-URL: Homepage, https://github.com/tracepipe/tracepipe
+Project-URL: Documentation, https://tracepipe.github.io/tracepipe/
+Project-URL: Repository, https://github.com/tracepipe/tracepipe.git
+Project-URL: Issues, https://github.com/tracepipe/tracepipe/issues
+Project-URL: Changelog, https://tracepipe.github.io/tracepipe/changelog/
+Author: Gauthier Piarrette
+License: MIT License
+        
+        Copyright (c) 2026 Gauthier Piarrette
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: data-engineering,data-lineage,data-quality,debugging,observability,pandas
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: pandas>=1.5.0
+Provides-Extra: all
+Requires-Dist: psutil>=5.9.0; extra == 'all'
+Requires-Dist: pyarrow>=10.0.0; extra == 'all'
+Provides-Extra: arrow
+Requires-Dist: pyarrow>=10.0.0; extra == 'arrow'
+Provides-Extra: dev
+Requires-Dist: black>=23.0.0; extra == 'dev'
+Requires-Dist: pre-commit>=3.5.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Requires-Dist: taskipy>=1.12.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
+Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Requires-Dist: pymdown-extensions>=10.0.0; extra == 'docs'
+Provides-Extra: memory
+Requires-Dist: psutil>=5.9.0; extra == 'memory'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+# TracePipe
+
+### Row-level data lineage for pandas pipelines
+
+**Know exactly where every row went, why values changed, and how your data transformed.**
+
+[![PyPI version](https://img.shields.io/pypi/v/tracepipe.svg)](https://pypi.org/project/tracepipe/)
+[![Python 3.9+](https://img.shields.io/pypi/pyversions/tracepipe.svg)](https://pypi.org/project/tracepipe/)
+[![CI](https://github.com/gauthierpiarrette/tracepipe/actions/workflows/ci.yml/badge.svg)](https://github.com/gauthierpiarrette/tracepipe/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/gauthierpiarrette/tracepipe/branch/main/graph/badge.svg)](https://codecov.io/gh/gauthierpiarrette/tracepipe)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://gauthierpiarrette.github.io/tracepipe/)
+
+[Getting Started](#getting-started) · [Documentation](https://gauthierpiarrette.github.io/tracepipe/) · [Examples](#real-world-example)
+
+</div>
+
+---
+
+## Why TracePipe?
+
+Data pipelines are black boxes. Rows vanish. Values change. You're left guessing.
+
+```python
+df = pd.read_csv("customers.csv")
+df = df.dropna()                      # Some rows disappear
+df = df.merge(regions, on="zip")      # New rows appear, some vanish
+df["income"] = df["income"].fillna(0) # Values change silently
+df = df[df["age"] >= 18]              # More rows gone
+# What happened to customer C-789? 🤷
+```
+
+**TracePipe gives you the complete audit trail — zero code changes required.**
+
+---
+
+## Getting Started
+
+```bash
+pip install tracepipe
+```
+
+```python
+import tracepipe as tp
+import pandas as pd
+
+tp.enable(mode="debug", watch=["income"])
+
+df = pd.read_csv("customers.csv")
+df = df.dropna()
+df["income"] = df["income"].fillna(0)
+df = df[df["age"] >= 18]
+
+tp.check(df)  # See what happened
+```
+
+```
+TracePipe Check: [OK] Pipeline healthy
+
+Retention: 847/1000 (84.7%)
+Dropped: 153 rows
+  • DataFrame.dropna: 42
+  • DataFrame.__getitem__[mask]: 111
+
+Value changes: 23 cells modified
+  • DataFrame.fillna: 23 (income)
+```
+
+That's it. **One import, full visibility.**
+
+---
+
+## Core API
+
+| Function | What it does |
+|----------|--------------|
+| `tp.enable()` | Start tracking |
+| `tp.check(df)` | Health check — retention, drops, changes |
+| `tp.trace(df, where={"id": "C-789"})` | Follow a row's complete journey |
+| `tp.why(df, col="income", row=5)` | Explain why a cell has its current value |
+| `tp.report(df, "audit.html")` | Export interactive HTML report |
+
+---
+
+## Key Features
+
+<table>
+<tr>
+<td width="50%">
+
+### 🔍 Zero-Code Instrumentation
+TracePipe patches pandas at runtime. Your existing code works unchanged.
+
+### 📊 Complete Provenance
+Track drops, transforms, merges, and cell-level changes with before/after values.
+
+</td>
+<td width="50%">
+
+### 🎯 Business-Key Lookups
+Find rows by their values: `tp.trace(df, where={"email": "alice@example.com"})`
+
+### ⚡ Production-Ready
+1.0-2.8x overhead (varies by operation). Tested on DataFrames up to 1M rows.
+
+</td>
+</tr>
+</table>
+
+---
+
+## Real-World Example
+
+```python
+import tracepipe as tp
+import pandas as pd
+
+tp.enable(mode="debug", watch=["age", "income", "label"])
+
+# Load and clean
+df = pd.read_csv("training_data.csv")
+df = df.dropna(subset=["label"])
+df["income"] = df["income"].fillna(df["income"].median())
+df = df[df["age"] >= 18]
+
+# Audit
+print(tp.check(df))
+```
+
+```
+Retention: 8234/10000 (82.3%)
+Dropped: 1766 rows
+  • DataFrame.dropna: 423
+  • DataFrame.__getitem__[mask]: 1343
+
+Value changes: 892 cells
+  • DataFrame.fillna: 892 (income)
+```
+
+```python
+# Why does this customer have a filled income?
+tp.why(df, col="income", where={"customer_id": "C-789"})
+```
+
+```
+Cell History: row 156, column 'income'
+  Current value: 45000.0
+  [i] Was null at step 1 (later recovered)
+
+  History (1 change):
+    None -> 45000.0
+      by: DataFrame.fillna
+```
+
+---
+
+## Two Modes
+
+| Mode | Use Case | What's Tracked |
+|------|----------|----------------|
+| **CI** (default) | Production pipelines | Step counts, retention rates, merge warnings |
+| **Debug** | Development | Full row history, cell diffs, merge parents, group membership |
+
+```python
+tp.enable(mode="ci")     # Lightweight
+tp.enable(mode="debug")  # Full lineage
+```
+
+---
+
+## What's Tracked
+
+| Operation | Coverage |
+|-----------|----------|
+| `dropna`, `drop_duplicates`, `query`, `df[mask]` | ✅ Full |
+| `fillna`, `replace`, `loc[]=`, `iloc[]=` | ✅ Full (cell diffs) |
+| `merge`, `join` | ✅ Full (parent tracking) |
+| `groupby().agg()` | ✅ Full (group membership) |
+| `sort_values`, `head`, `tail`, `sample` | ✅ Full |
+| `apply`, `pipe` | ⚠️ Partial |
+
+---
+
+## Data Quality Contracts
+
+```python
+(tp.contract()
+    .expect_unique("customer_id")
+    .expect_no_nulls("email")
+    .expect_retention(min_rate=0.9)
+    .check(df)
+    .raise_if_failed())
+```
+
+---
+
+## Documentation
+
+📚 **[Full Documentation](https://gauthierpiarrette.github.io/tracepipe/)**
+
+- [Quickstart](https://gauthierpiarrette.github.io/tracepipe/getting-started/quickstart/)
+- [User Guide](https://gauthierpiarrette.github.io/tracepipe/guide/concepts/)
+- [API Reference](https://gauthierpiarrette.github.io/tracepipe/api/)
+- [Examples](https://gauthierpiarrette.github.io/tracepipe/examples/ml-pipeline/)
+
+---
+
+## Contributing
+
+```bash
+git clone https://github.com/gauthierpiarrette/tracepipe.git
+cd tracepipe
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+See [CONTRIBUTING](https://gauthierpiarrette.github.io/tracepipe/contributing/) for guidelines.
+
+---
+
+## License
+
+MIT License. See [LICENSE](LICENSE).
+
+---
+
+<div align="center">
+
+**Stop guessing where your rows went.**
+
+```bash
+pip install tracepipe
+```
+
+⭐ Star us on GitHub if TracePipe helps your data work!
+
+</div>