leakprobe 0.1.0__py3-none-any.whl

leakprobe/__init__.py

@@ -0,0 +1,19 @@
+"""leakprobe -- find features that read data they were never supposed to see.
+
+    import leakprobe as lc
+
+    report = lc.check(
+        compute=build_features,
+        sources={"events": events, "profiles": profiles},
+        timestamps={"events": "occurred_at", "profiles": "updated_at"},
+        declared={"rolling_7d_spend": ["events"]},
+    )
+    report.raise_for_leaks()
+"""
+
+from .core import check
+from .perturb import advance, delay, use_column
+from .report import Finding, Report
+
+__all__ = ["check", "delay", "advance", "use_column", "Report", "Finding"]
+__version__ = "0.1.0"

leakprobe/core.py

@@ -0,0 +1,150 @@
+"""Find features that read a source they never declared.
+
+The test needs no ground truth. It needs one change the answer must be
+invariant to: move when a source became knowable, recompute, and read the list
+of features that flinched.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+
+import pandas as pd
+
+from .perturb import delay
+from .report import BYPASS, LEAK, OK, Finding, Report
+
+__all__ = ["check"]
+
+Sources = Mapping[str, pd.DataFrame]
+Compute = Callable[[Sources], pd.DataFrame]
+
+
+def _as_frame(result: object, label: str) -> pd.DataFrame:
+    if isinstance(result, pd.Series):
+        return result.to_frame()
+    if isinstance(result, pd.DataFrame):
+        return result
+    raise TypeError(f"{label} must return a DataFrame or Series, got {type(result).__name__}")
+
+
+def _changed(left: pd.Series, right: pd.Series, tolerance: float) -> tuple[bool, float]:
+    """Did this column move? NaN in the same place counts as unchanged."""
+    both_null = left.isna() & right.isna()
+    if (left.isna() != right.isna()).any():
+        return True, float("inf")
+    if pd.api.types.is_numeric_dtype(left) and pd.api.types.is_numeric_dtype(right):
+        gap = (left - right).abs()
+        gap = gap[~both_null]
+        worst = float(gap.max()) if len(gap) else 0.0
+        return worst > tolerance, worst
+    unequal = (left != right) & ~both_null
+    return bool(unequal.any()), float(unequal.sum())
+
+
+def check(
+    compute: Compute,
+    sources: Sources,
+    timestamps: Mapping[str, str],
+    declared: Mapping[str, Sequence[str]],
+    *,
+    by: pd.Timedelta = pd.Timedelta(days=30),
+    tolerance: float = 0.0,
+    perturb: Callable[[pd.DataFrame, str, pd.Timedelta], pd.DataFrame] = delay,
+) -> Report:
+    """Perturb each source's availability in turn; report features that moved.
+
+    Args:
+        compute: builds your features. Takes the sources dict, returns a frame
+            whose columns are features. Must be deterministic -- this is checked.
+        sources: name -> raw table. Each carries an availability timestamp.
+        timestamps: source name -> the column holding that timestamp, or None for
+            a static table with no clock, which is skipped.
+        declared: feature name -> the sources it is supposed to read. A feature
+            absent from this mapping is treated as declaring nothing, which is
+            the strictest reading and usually what you want for a new pipeline.
+        by: how far to move availability. Large enough to bite, small enough to
+            leave rows in the window.
+        tolerance: treat changes at or below this as no change. Leave at 0.0
+            unless your pipeline has genuine floating-point nondeterminism, and
+            prefer to fix that instead.
+        perturb: the perturbation. `delay` removes information and is the safe
+            default; see `leakprobe.perturb` for others.
+
+    Returns:
+        A Report. `report.leaks` is the list of undeclared dependencies, and
+        `report.raise_for_leaks()` turns them into a test failure.
+    """
+    absent = set(sources) - set(timestamps)
+    if absent:
+        raise KeyError(
+            f"No availability column given for source(s): {sorted(absent)}. "
+            "Pass None for a static table that has no clock."
+        )
+
+    unknown = {s for deps in declared.values() for s in deps} - set(sources)
+    if unknown:
+        raise KeyError(f"declared refers to unknown source(s): {sorted(unknown)}")
+
+    baseline = _as_frame(compute(sources), "compute")
+    control = _as_frame(compute(sources), "compute")
+
+    notes: list[str] = []
+    if list(baseline.columns) != list(control.columns):
+        raise RuntimeError(
+            "compute() returned different columns on two identical calls. The test "
+            "cannot separate a leak from nondeterminism until that is fixed."
+        )
+    unstable = [c for c in baseline.columns if _changed(baseline[c], control[c], tolerance)[0]]
+    if unstable:
+        raise RuntimeError(
+            "compute() is not deterministic: "
+            + ", ".join(unstable[:5])
+            + (" ..." if len(unstable) > 5 else "")
+            + ". Seed it, or every result below is noise."
+        )
+
+    features = [str(c) for c in baseline.columns]
+    findings: list[Finding] = []
+
+    for source in sources:
+        if timestamps[source] is None:
+            notes.append(
+                f"note: {source!r} has no availability column, so nothing about it was "
+                f"perturbed. Dependencies on it are untested."
+            )
+            continue
+        moved_frame = _as_frame(
+            compute({**sources, source: perturb(sources[source], timestamps[source], by)}),
+            "compute",
+        )
+        if list(moved_frame.columns) != list(baseline.columns):
+            notes.append(
+                f"note: perturbing {source!r} changed the feature set itself, which is "
+                f"a stronger dependency than this test is designed to describe."
+            )
+            continue
+
+        for column, feature in zip(baseline.columns, features):
+            moved, worst = _changed(baseline[column], moved_frame[column], tolerance)
+            is_declared = source in declared.get(feature, ())
+            if moved and not is_declared:
+                kind = LEAK
+            elif is_declared and not moved:
+                kind = BYPASS
+            else:
+                kind = OK
+            findings.append(
+                Finding(
+                    feature=feature,
+                    source=source,
+                    kind=kind,
+                    declared=is_declared,
+                    moved=moved,
+                    max_abs_change=worst,
+                )
+            )
+
+    return Report(
+        findings=findings, features=features, sources=list(sources), notes=notes
+    )

leakprobe/perturb.py

@@ -0,0 +1,64 @@
+"""Ways to move when a source becomes knowable.
+
+A perturbation changes *availability*, never values. That distinction is what
+makes the test sound: if a feature's number changes after a perturbation, the
+feature read something whose timing moved, and timing is the only thing that
+moved.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+__all__ = ["delay", "advance", "use_column"]
+
+
+def _check(frame: pd.DataFrame, column: str) -> None:
+    if column not in frame.columns:
+        raise KeyError(
+            f"No availability column {column!r}. Columns present: {list(frame.columns)}"
+        )
+    if not pd.api.types.is_datetime64_any_dtype(frame[column]):
+        raise TypeError(
+            f"Availability column {column!r} is {frame[column].dtype}, not a datetime. "
+            "Parse it with pd.to_datetime first."
+        )
+
+
+def delay(frame: pd.DataFrame, column: str, by: pd.Timedelta) -> pd.DataFrame:
+    """The same rows, knowable `by` later.
+
+    The safe default. Delaying a source can only remove information, so a
+    feature that legitimately reads it will change and one that does not cannot.
+    """
+    _check(frame, column)
+    out = frame.copy()
+    out[column] = out[column] + by
+    return out
+
+
+def advance(frame: pd.DataFrame, column: str, by: pd.Timedelta) -> pd.DataFrame:
+    """The same rows, knowable `by` earlier -- look-ahead, injected on purpose.
+
+    Use when you want to measure what a leak would be *worth* rather than
+    whether one exists.
+    """
+    _check(frame, column)
+    out = frame.copy()
+    out[column] = out[column] - by
+    return out
+
+
+def use_column(frame: pd.DataFrame, column: str, source: str) -> pd.DataFrame:
+    """Availability taken from another column.
+
+    Models the specific mistake of treating a record as knowable when the period
+    it describes ended, rather than when it was published: `use_column(facts,
+    "available_at", "period_end")`.
+    """
+    _check(frame, column)
+    if source not in frame.columns:
+        raise KeyError(f"No column {source!r} to take availability from.")
+    out = frame.copy()
+    out[column] = out[source]
+    return out

leakprobe/report.py

@@ -0,0 +1,109 @@
+"""The verdict, and how it prints."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+__all__ = ["Finding", "Report"]
+
+LEAK = "leak"
+BYPASS = "bypass"
+OK = "ok"
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One (feature, source) pair, and what the perturbation did to it."""
+
+    feature: str
+    source: str
+    kind: str
+    declared: bool
+    moved: bool
+    max_abs_change: float
+
+    def __str__(self) -> str:
+        if self.kind == LEAK:
+            return (
+                f"{self.feature} moved when {self.source} was perturbed, and does not "
+                f"declare it (max change {self.max_abs_change:.3g})"
+            )
+        if self.kind == BYPASS:
+            return (
+                f"{self.feature} declares {self.source} but did not move when its clock "
+                f"did -- it reads the source without consulting availability, or the "
+                f"declaration is stale"
+            )
+        return f"{self.feature} / {self.source}: as declared"
+
+
+@dataclass
+class Report:
+    findings: list[Finding] = field(default_factory=list)
+    features: list[str] = field(default_factory=list)
+    sources: list[str] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def leaks(self) -> list[Finding]:
+        """Undeclared dependencies. These are the bugs."""
+        return [f for f in self.findings if f.kind == LEAK]
+
+    @property
+    def bypassed(self) -> list[Finding]:
+        """Declared dependencies that did not respond to their own clock.
+
+        Either the declaration is stale, or the feature reaches the source by a
+        path that ignores its availability column -- which is how look-ahead
+        usually gets in. Not a failure on its own; worth reading.
+        """
+        return [f for f in self.findings if f.kind == BYPASS]
+
+    @property
+    def clean(self) -> bool:
+        return not self.leaks
+
+    def raise_for_leaks(self) -> None:
+        """For use in a test suite: turn a leak into a failure."""
+        if self.leaks:
+            listed = "\n".join(f"  - {f}" for f in self.leaks)
+            raise AssertionError(f"{len(self.leaks)} undeclared dependencies:\n{listed}")
+
+    def __str__(self) -> str:
+        width = max((len(f) for f in self.features), default=7)
+        lines = [
+            f"{len(self.features)} features x {len(self.sources)} sources",
+            "",
+            f"{'feature':<{width}}  " + "  ".join(f"{s:>12}" for s in self.sources),
+        ]
+        by_pair = {(f.feature, f.source): f for f in self.findings}
+        for feature in self.features:
+            cells = []
+            for source in self.sources:
+                found = by_pair.get((feature, source))
+                if found is None:
+                    cells.append(f"{'-':>12}")
+                elif found.kind == LEAK:
+                    cells.append(f"{'LEAK':>12}")
+                elif found.kind == BYPASS:
+                    cells.append(f"{'bypass?':>12}")
+                elif found.declared:
+                    cells.append(f"{'reads it':>12}")
+                else:
+                    cells.append(f"{'exactly 0':>12}")
+            lines.append(f"{feature:<{width}}  " + "  ".join(cells))
+
+        lines.append("")
+        if self.leaks:
+            lines.append(f"{len(self.leaks)} undeclared dependencies:")
+            lines += [f"  - {f}" for f in self.leaks]
+        else:
+            lines.append("No undeclared dependencies.")
+        if self.bypassed:
+            lines.append("")
+            lines.append("Declared, but did not respond to the source's clock:")
+            lines += [f"  - {f}" for f in self.bypassed]
+        if self.notes:
+            lines.append("")
+            lines += self.notes
+        return "\n".join(lines)

leakprobe-0.1.0.dist-info/METADATA

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: leakprobe
+Version: 0.1.0
+Summary: Find features that read data they were never supposed to see. Temporal leakage detection by metamorphic testing.
+Author: Raunak Sood
+License: MIT
+Project-URL: Homepage, https://github.com/quantraunak/leakprobe
+Project-URL: Issues, https://github.com/quantraunak/leakprobe/issues
+Keywords: leakage,data-leakage,machine-learning,testing,time-series,point-in-time,feature-engineering
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: numpy>=1.26; extra == "dev"
+Provides-Extra: examples
+Requires-Dist: numpy>=1.26; extra == "examples"
+Requires-Dist: openpyxl>=3.1; extra == "examples"
+Dynamic: license-file
+
+# leakprobe
+
+[![CI](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml/badge.svg)](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/leakprobe.svg)](https://pypi.org/project/leakprobe/) [![Python](https://img.shields.io/pypi/pyversions/leakprobe.svg)](https://pypi.org/project/leakprobe/)
+
+
+**Find features that read data they were never supposed to see.**
+
+Temporal leakage is the most expensive quiet bug in applied ML. A feature reads
+something that wasn't knowable yet, nothing throws, no number looks implausible,
+and your model gets better. You find out in production, if you find out at all.
+
+`leakprobe` finds it without needing to know the right answer. It needs one
+thing you already know: **a change to your data that your features must be
+invariant to.**
+
+```bash
+pip install leakprobe
+```
+
+```python
+import leakprobe as lp
+
+report = lp.check(
+    compute=build_features,                  # (sources) -> DataFrame of features
+    sources={"events": events, "tickets": tickets},
+    timestamps={"events": "occurred_at", "tickets": "resolved_at"},
+    declared={
+        "total_spend":      ["events"],
+        "event_count":      ["events"],
+        "tickets_resolved": ["tickets"],
+        "avg_severity":     ["tickets"],
+    },
+)
+report.raise_for_leaks()      # fails your test suite if anything leaked
+```
+
+```
+4 features x 2 sources
+
+feature                 events       tickets
+total_spend           reads it     exactly 0
+event_count           reads it     exactly 0
+tickets_resolved     exactly 0      reads it
+avg_severity         exactly 0       bypass?
+
+No undeclared dependencies.
+
+Declared, but did not respond to the source's clock:
+  - avg_severity declares tickets but did not move when its clock did -- it
+    reads the source without consulting availability, or the declaration is stale
+```
+
+That last line is a real bug. `avg_severity` filters tickets on `opened_at`
+instead of `resolved_at`, so tickets that were still open at scoring time leak
+in — exactly the ones that predict churn. Nothing about the code looks wrong.
+
+## Try it on real data
+
+```bash
+pip install leakprobe openpyxl
+python examples/online_retail.py
+```
+
+The UCI Online Retail set: 397,924 real orders and 8,905 returns across 4,372
+customers of a UK gift retailer, 2010-2011. Six ordinary per-customer features
+built as of a cutoff, two of them wrong in the two ways temporal leakage
+actually happens. Neither raises. Both are caught:
+
+```
+feature               orders       returns
+total_spend         reads it     exactly 0
+order_count         reads it     exactly 0
+recency_days        reads it     exactly 0
+return_count       exactly 0      reads it
+avg_unit_price       bypass?     exactly 0
+net_spend           reads it          LEAK
+
+1 undeclared dependencies:
+  - net_spend moved when returns was perturbed, and does not declare it (max change 7.46e+03)
+
+Declared, but did not respond to the source's clock:
+  - avg_unit_price declares orders but did not move when its clock did
+```
+
+`avg_unit_price` is missing one cutoff filter, so it averages invoices that had
+not happened yet. `net_spend` reaches into the returns table without declaring
+it, silently inheriting that table's latency. One dropped subscript and one
+undeclared read -- the two shapes this bug takes in production.
+
+## How it works
+
+Four steps, and no ground truth anywhere in them.
+
+1. **Move when a source became knowable.** Not its values — only its
+   availability timestamp. `delay` pushes it later, which can only ever remove
+   information.
+2. **Recompute every feature.**
+3. **Compare, exactly.** A feature that genuinely cannot read that source gets
+   the identical input arrays through the identical code and returns bit-identical
+   floats. Its difference is `0.0`, not `1e-15`. So any movement at all is proof
+   of a dependency, not a number you have to squint at.
+4. **Check what moved against what you declared.** Anything in one list and not
+   the other is the finding.
+
+Two kinds of finding:
+
+- **`leak`** — moved, but doesn't declare the source. It has a dependency you
+  didn't know about. In a temporal pipeline, an unknown dependency on *when*
+  data arrived is look-ahead.
+- **`bypass?`** — declares the source but didn't move when that source's clock
+  did. It's reaching the data by a path that ignores availability, which is how
+  look-ahead usually gets in. Not a failure on its own. Worth reading.
+
+Before any of that, `check` runs `compute` twice on untouched inputs and refuses
+to continue if the two runs disagree. A nondeterministic pipeline makes every
+result below it noise, so that's a hard error rather than a warning.
+
+## Declaring nothing
+
+You don't have to write the `declared` map. Leave it out and every real timing
+dependency is reported:
+
+```python
+report = lp.check(compute, sources, timestamps, declared={})
+for f in report.leaks:
+    print(f.feature, "reads", f.source)
+```
+
+That's the fastest way to answer "what does this pipeline actually depend on"
+for code you inherited, which is usually a shorter list than the author believed.
+
+## In CI
+
+```python
+def test_no_temporal_leakage():
+    lp.check(build_features, SOURCES, TIMESTAMPS, DECLARED).raise_for_leaks()
+```
+
+The declaration map becomes the thing code review argues about, which is where
+that argument belongs.
+
+## Perturbations
+
+| | |
+|---|---|
+| `delay(frame, col, by)` | knowable later. The safe default: removes information only. |
+| `advance(frame, col, by)` | knowable earlier. Injects look-ahead on purpose, to measure what a leak is worth. |
+| `use_column(frame, col, other)` | availability taken from another column. Models "treated as knowable when the period ended, not when it was published." |
+
+```python
+report = lp.check(..., perturb=lp.advance, by=pd.Timedelta(days=90))
+```
+
+## What it will not catch
+
+**Dependencies that don't flow through a timestamp.** The perturbation moves
+availability, so it reveals as-of joins, merges on a date, and windows anchored
+to one. A feature that reads a source's values with no reference to when they
+arrived is invariant to it and will not be flagged. In practice this costs less
+than it sounds: look-ahead *is* a dependency on timing, so the leaks worth
+catching are the detectable ones. The boundary is asserted in the test suite so
+it can't quietly stop being true.
+
+**Sources with no clock.** Pass `None` and static tables are skipped, with a
+note saying dependencies on them went untested.
+
+**Leakage across rows rather than time** — target encoding fit on the full
+dataset, a scaler fit before the split. Different bug, different tool.
+
+## Where this came from
+
+The technique is metamorphic testing, which software testing has used for
+decades and research code almost never does. This is an extraction of a check
+built for a quantitative finance pipeline, where the sources are stock prices
+and regulatory filings and the question is whether a factor read an earnings
+figure before it was published.
+
+It caught a real one. A factor called `turnover_1m` was classified as
+price-and-volume — it's built from volume, it lives in `price.py`, and every
+human who looked at it filed it under prices. It divides by shares outstanding,
+which comes off a filing. It was the only member of its group that moved when
+the filing calendar shifted, while eleven genuine price factors held at exactly
+zero. That measurement is written up in
+[bias-fingerprints](https://github.com/quantraunak/bias-fingerprints).
+
+## Install
+
+```bash
+pip install leakprobe          # pandas is the only dependency
+```
+
+Python 3.10+.
+
+## Licence
+
+MIT.

leakprobe-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+leakprobe/__init__.py,sha256=D1M2m_5CJRZ_I3W146fTQweiGgLXAboX37KZthBtnJg,589
+leakprobe/core.py,sha256=QMoIUXZRootv4DAF4S8U6rTPX6YdBLvwXv5vxWeLXhw,5869
+leakprobe/perturb.py,sha256=Crqkx7rICrNhg-f7bBqqXFGYZYL2byYbBY7UicEFrx0,2142
+leakprobe/report.py,sha256=7XMoUHJTy4N3von3kABCBf2fMRgU_f9rPetpUgao4Tw,3799
+leakprobe-0.1.0.dist-info/licenses/LICENSE,sha256=fYY74xctr_i5pdD0yvpdpWnLU9Mk4BH3_Vg6qMU13C0,1068
+leakprobe-0.1.0.dist-info/METADATA,sha256=eDU3vaU2PnZ_z2rjP3quuadkGy7jVocPS_egNfgM564,8822
+leakprobe-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+leakprobe-0.1.0.dist-info/top_level.txt,sha256=oz6lJczqUf_MZLxS6ZoOEHxt-pPrym9QH09DCawWf8c,10
+leakprobe-0.1.0.dist-info/RECORD,,

leakprobe-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

leakprobe-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Raunak Sood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

leakprobe-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+leakprobe