leakprobe 0.1.0__tar.gz

leakprobe-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Raunak Sood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

leakprobe-0.1.0/PKG-INFO

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: leakprobe
+Version: 0.1.0
+Summary: Find features that read data they were never supposed to see. Temporal leakage detection by metamorphic testing.
+Author: Raunak Sood
+License: MIT
+Project-URL: Homepage, https://github.com/quantraunak/leakprobe
+Project-URL: Issues, https://github.com/quantraunak/leakprobe/issues
+Keywords: leakage,data-leakage,machine-learning,testing,time-series,point-in-time,feature-engineering
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: numpy>=1.26; extra == "dev"
+Provides-Extra: examples
+Requires-Dist: numpy>=1.26; extra == "examples"
+Requires-Dist: openpyxl>=3.1; extra == "examples"
+Dynamic: license-file
+
+# leakprobe
+
+[![CI](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml/badge.svg)](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/leakprobe.svg)](https://pypi.org/project/leakprobe/) [![Python](https://img.shields.io/pypi/pyversions/leakprobe.svg)](https://pypi.org/project/leakprobe/)
+
+
+**Find features that read data they were never supposed to see.**
+
+Temporal leakage is the most expensive quiet bug in applied ML. A feature reads
+something that wasn't knowable yet, nothing throws, no number looks implausible,
+and your model gets better. You find out in production, if you find out at all.
+
+`leakprobe` finds it without needing to know the right answer. It needs one
+thing you already know: **a change to your data that your features must be
+invariant to.**
+
+```bash
+pip install leakprobe
+```
+
+```python
+import leakprobe as lp
+
+report = lp.check(
+    compute=build_features,                  # (sources) -> DataFrame of features
+    sources={"events": events, "tickets": tickets},
+    timestamps={"events": "occurred_at", "tickets": "resolved_at"},
+    declared={
+        "total_spend":      ["events"],
+        "event_count":      ["events"],
+        "tickets_resolved": ["tickets"],
+        "avg_severity":     ["tickets"],
+    },
+)
+report.raise_for_leaks()      # fails your test suite if anything leaked
+```
+
+```
+4 features x 2 sources
+
+feature                 events       tickets
+total_spend           reads it     exactly 0
+event_count           reads it     exactly 0
+tickets_resolved     exactly 0      reads it
+avg_severity         exactly 0       bypass?
+
+No undeclared dependencies.
+
+Declared, but did not respond to the source's clock:
+  - avg_severity declares tickets but did not move when its clock did -- it
+    reads the source without consulting availability, or the declaration is stale
+```
+
+That last line is a real bug. `avg_severity` filters tickets on `opened_at`
+instead of `resolved_at`, so tickets that were still open at scoring time leak
+in — exactly the ones that predict churn. Nothing about the code looks wrong.
+
+## Try it on real data
+
+```bash
+pip install leakprobe openpyxl
+python examples/online_retail.py
+```
+
+The UCI Online Retail set: 397,924 real orders and 8,905 returns across 4,372
+customers of a UK gift retailer, 2010-2011. Six ordinary per-customer features
+built as of a cutoff, two of them wrong in the two ways temporal leakage
+actually happens. Neither raises. Both are caught:
+
+```
+feature               orders       returns
+total_spend         reads it     exactly 0
+order_count         reads it     exactly 0
+recency_days        reads it     exactly 0
+return_count       exactly 0      reads it
+avg_unit_price       bypass?     exactly 0
+net_spend           reads it          LEAK
+
+1 undeclared dependencies:
+  - net_spend moved when returns was perturbed, and does not declare it (max change 7.46e+03)
+
+Declared, but did not respond to the source's clock:
+  - avg_unit_price declares orders but did not move when its clock did
+```
+
+`avg_unit_price` is missing one cutoff filter, so it averages invoices that had
+not happened yet. `net_spend` reaches into the returns table without declaring
+it, silently inheriting that table's latency. One dropped subscript and one
+undeclared read -- the two shapes this bug takes in production.
+
+## How it works
+
+Four steps, and no ground truth anywhere in them.
+
+1. **Move when a source became knowable.** Not its values — only its
+   availability timestamp. `delay` pushes it later, which can only ever remove
+   information.
+2. **Recompute every feature.**
+3. **Compare, exactly.** A feature that genuinely cannot read that source gets
+   the identical input arrays through the identical code and returns bit-identical
+   floats. Its difference is `0.0`, not `1e-15`. So any movement at all is proof
+   of a dependency, not a number you have to squint at.
+4. **Check what moved against what you declared.** Anything in one list and not
+   the other is the finding.
+
+Two kinds of finding:
+
+- **`leak`** — moved, but doesn't declare the source. It has a dependency you
+  didn't know about. In a temporal pipeline, an unknown dependency on *when*
+  data arrived is look-ahead.
+- **`bypass?`** — declares the source but didn't move when that source's clock
+  did. It's reaching the data by a path that ignores availability, which is how
+  look-ahead usually gets in. Not a failure on its own. Worth reading.
+
+Before any of that, `check` runs `compute` twice on untouched inputs and refuses
+to continue if the two runs disagree. A nondeterministic pipeline makes every
+result below it noise, so that's a hard error rather than a warning.
+
+## Declaring nothing
+
+You don't have to write the `declared` map. Leave it out and every real timing
+dependency is reported:
+
+```python
+report = lp.check(compute, sources, timestamps, declared={})
+for f in report.leaks:
+    print(f.feature, "reads", f.source)
+```
+
+That's the fastest way to answer "what does this pipeline actually depend on"
+for code you inherited, which is usually a shorter list than the author believed.
+
+## In CI
+
+```python
+def test_no_temporal_leakage():
+    lp.check(build_features, SOURCES, TIMESTAMPS, DECLARED).raise_for_leaks()
+```
+
+The declaration map becomes the thing code review argues about, which is where
+that argument belongs.
+
+## Perturbations
+
+| | |
+|---|---|
+| `delay(frame, col, by)` | knowable later. The safe default: removes information only. |
+| `advance(frame, col, by)` | knowable earlier. Injects look-ahead on purpose, to measure what a leak is worth. |
+| `use_column(frame, col, other)` | availability taken from another column. Models "treated as knowable when the period ended, not when it was published." |
+
+```python
+report = lp.check(..., perturb=lp.advance, by=pd.Timedelta(days=90))
+```
+
+## What it will not catch
+
+**Dependencies that don't flow through a timestamp.** The perturbation moves
+availability, so it reveals as-of joins, merges on a date, and windows anchored
+to one. A feature that reads a source's values with no reference to when they
+arrived is invariant to it and will not be flagged. In practice this costs less
+than it sounds: look-ahead *is* a dependency on timing, so the leaks worth
+catching are the detectable ones. The boundary is asserted in the test suite so
+it can't quietly stop being true.
+
+**Sources with no clock.** Pass `None` and static tables are skipped, with a
+note saying dependencies on them went untested.
+
+**Leakage across rows rather than time** — target encoding fit on the full
+dataset, a scaler fit before the split. Different bug, different tool.
+
+## Where this came from
+
+The technique is metamorphic testing, which software testing has used for
+decades and research code almost never does. This is an extraction of a check
+built for a quantitative finance pipeline, where the sources are stock prices
+and regulatory filings and the question is whether a factor read an earnings
+figure before it was published.
+
+It caught a real one. A factor called `turnover_1m` was classified as
+price-and-volume — it's built from volume, it lives in `price.py`, and every
+human who looked at it filed it under prices. It divides by shares outstanding,
+which comes off a filing. It was the only member of its group that moved when
+the filing calendar shifted, while eleven genuine price factors held at exactly
+zero. That measurement is written up in
+[bias-fingerprints](https://github.com/quantraunak/bias-fingerprints).
+
+## Install
+
+```bash
+pip install leakprobe          # pandas is the only dependency
+```
+
+Python 3.10+.
+
+## Licence
+
+MIT.

leakprobe-0.1.0/README.md

@@ -0,0 +1,196 @@
+# leakprobe
+
+[![CI](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml/badge.svg)](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/leakprobe.svg)](https://pypi.org/project/leakprobe/) [![Python](https://img.shields.io/pypi/pyversions/leakprobe.svg)](https://pypi.org/project/leakprobe/)
+
+
+**Find features that read data they were never supposed to see.**
+
+Temporal leakage is the most expensive quiet bug in applied ML. A feature reads
+something that wasn't knowable yet, nothing throws, no number looks implausible,
+and your model gets better. You find out in production, if you find out at all.
+
+`leakprobe` finds it without needing to know the right answer. It needs one
+thing you already know: **a change to your data that your features must be
+invariant to.**
+
+```bash
+pip install leakprobe
+```
+
+```python
+import leakprobe as lp
+
+report = lp.check(
+    compute=build_features,                  # (sources) -> DataFrame of features
+    sources={"events": events, "tickets": tickets},
+    timestamps={"events": "occurred_at", "tickets": "resolved_at"},
+    declared={
+        "total_spend":      ["events"],
+        "event_count":      ["events"],
+        "tickets_resolved": ["tickets"],
+        "avg_severity":     ["tickets"],
+    },
+)
+report.raise_for_leaks()      # fails your test suite if anything leaked
+```
+
+```
+4 features x 2 sources
+
+feature                 events       tickets
+total_spend           reads it     exactly 0
+event_count           reads it     exactly 0
+tickets_resolved     exactly 0      reads it
+avg_severity         exactly 0       bypass?
+
+No undeclared dependencies.
+
+Declared, but did not respond to the source's clock:
+  - avg_severity declares tickets but did not move when its clock did -- it
+    reads the source without consulting availability, or the declaration is stale
+```
+
+That last line is a real bug. `avg_severity` filters tickets on `opened_at`
+instead of `resolved_at`, so tickets that were still open at scoring time leak
+in — exactly the ones that predict churn. Nothing about the code looks wrong.
+
+## Try it on real data
+
+```bash
+pip install leakprobe openpyxl
+python examples/online_retail.py
+```
+
+The UCI Online Retail set: 397,924 real orders and 8,905 returns across 4,372
+customers of a UK gift retailer, 2010-2011. Six ordinary per-customer features
+built as of a cutoff, two of them wrong in the two ways temporal leakage
+actually happens. Neither raises. Both are caught:
+
+```
+feature               orders       returns
+total_spend         reads it     exactly 0
+order_count         reads it     exactly 0
+recency_days        reads it     exactly 0
+return_count       exactly 0      reads it
+avg_unit_price       bypass?     exactly 0
+net_spend           reads it          LEAK
+
+1 undeclared dependencies:
+  - net_spend moved when returns was perturbed, and does not declare it (max change 7.46e+03)
+
+Declared, but did not respond to the source's clock:
+  - avg_unit_price declares orders but did not move when its clock did
+```
+
+`avg_unit_price` is missing one cutoff filter, so it averages invoices that had
+not happened yet. `net_spend` reaches into the returns table without declaring
+it, silently inheriting that table's latency. One dropped subscript and one
+undeclared read -- the two shapes this bug takes in production.
+
+## How it works
+
+Four steps, and no ground truth anywhere in them.
+
+1. **Move when a source became knowable.** Not its values — only its
+   availability timestamp. `delay` pushes it later, which can only ever remove
+   information.
+2. **Recompute every feature.**
+3. **Compare, exactly.** A feature that genuinely cannot read that source gets
+   the identical input arrays through the identical code and returns bit-identical
+   floats. Its difference is `0.0`, not `1e-15`. So any movement at all is proof
+   of a dependency, not a number you have to squint at.
+4. **Check what moved against what you declared.** Anything in one list and not
+   the other is the finding.
+
+Two kinds of finding:
+
+- **`leak`** — moved, but doesn't declare the source. It has a dependency you
+  didn't know about. In a temporal pipeline, an unknown dependency on *when*
+  data arrived is look-ahead.
+- **`bypass?`** — declares the source but didn't move when that source's clock
+  did. It's reaching the data by a path that ignores availability, which is how
+  look-ahead usually gets in. Not a failure on its own. Worth reading.
+
+Before any of that, `check` runs `compute` twice on untouched inputs and refuses
+to continue if the two runs disagree. A nondeterministic pipeline makes every
+result below it noise, so that's a hard error rather than a warning.
+
+## Declaring nothing
+
+You don't have to write the `declared` map. Leave it out and every real timing
+dependency is reported:
+
+```python
+report = lp.check(compute, sources, timestamps, declared={})
+for f in report.leaks:
+    print(f.feature, "reads", f.source)
+```
+
+That's the fastest way to answer "what does this pipeline actually depend on"
+for code you inherited, which is usually a shorter list than the author believed.
+
+## In CI
+
+```python
+def test_no_temporal_leakage():
+    lp.check(build_features, SOURCES, TIMESTAMPS, DECLARED).raise_for_leaks()
+```
+
+The declaration map becomes the thing code review argues about, which is where
+that argument belongs.
+
+## Perturbations
+
+| | |
+|---|---|
+| `delay(frame, col, by)` | knowable later. The safe default: removes information only. |
+| `advance(frame, col, by)` | knowable earlier. Injects look-ahead on purpose, to measure what a leak is worth. |
+| `use_column(frame, col, other)` | availability taken from another column. Models "treated as knowable when the period ended, not when it was published." |
+
+```python
+report = lp.check(..., perturb=lp.advance, by=pd.Timedelta(days=90))
+```
+
+## What it will not catch
+
+**Dependencies that don't flow through a timestamp.** The perturbation moves
+availability, so it reveals as-of joins, merges on a date, and windows anchored
+to one. A feature that reads a source's values with no reference to when they
+arrived is invariant to it and will not be flagged. In practice this costs less
+than it sounds: look-ahead *is* a dependency on timing, so the leaks worth
+catching are the detectable ones. The boundary is asserted in the test suite so
+it can't quietly stop being true.
+
+**Sources with no clock.** Pass `None` and static tables are skipped, with a
+note saying dependencies on them went untested.
+
+**Leakage across rows rather than time** — target encoding fit on the full
+dataset, a scaler fit before the split. Different bug, different tool.
+
+## Where this came from
+
+The technique is metamorphic testing, which software testing has used for
+decades and research code almost never does. This is an extraction of a check
+built for a quantitative finance pipeline, where the sources are stock prices
+and regulatory filings and the question is whether a factor read an earnings
+figure before it was published.
+
+It caught a real one. A factor called `turnover_1m` was classified as
+price-and-volume — it's built from volume, it lives in `price.py`, and every
+human who looked at it filed it under prices. It divides by shares outstanding,
+which comes off a filing. It was the only member of its group that moved when
+the filing calendar shifted, while eleven genuine price factors held at exactly
+zero. That measurement is written up in
+[bias-fingerprints](https://github.com/quantraunak/bias-fingerprints).
+
+## Install
+
+```bash
+pip install leakprobe          # pandas is the only dependency
+```
+
+Python 3.10+.
+
+## Licence
+
+MIT.

leakprobe-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "leakprobe"
+version = "0.1.0"
+description = "Find features that read data they were never supposed to see. Temporal leakage detection by metamorphic testing."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Raunak Sood" }]
+keywords = ["leakage", "data-leakage", "machine-learning", "testing", "time-series", "point-in-time", "feature-engineering"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = ["pandas>=2.0"]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.4", "numpy>=1.26"]
+examples = ["numpy>=1.26", "openpyxl>=3.1"]
+
+[project.urls]
+Homepage = "https://github.com/quantraunak/leakprobe"
+Issues = "https://github.com/quantraunak/leakprobe/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

leakprobe-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

leakprobe-0.1.0/src/leakprobe/__init__.py

@@ -0,0 +1,19 @@
+"""leakprobe -- find features that read data they were never supposed to see.
+
+    import leakprobe as lc
+
+    report = lc.check(
+        compute=build_features,
+        sources={"events": events, "profiles": profiles},
+        timestamps={"events": "occurred_at", "profiles": "updated_at"},
+        declared={"rolling_7d_spend": ["events"]},
+    )
+    report.raise_for_leaks()
+"""
+
+from .core import check
+from .perturb import advance, delay, use_column
+from .report import Finding, Report
+
+__all__ = ["check", "delay", "advance", "use_column", "Report", "Finding"]
+__version__ = "0.1.0"

leakprobe-0.1.0/src/leakprobe/core.py

@@ -0,0 +1,150 @@
+"""Find features that read a source they never declared.
+
+The test needs no ground truth. It needs one change the answer must be
+invariant to: move when a source became knowable, recompute, and read the list
+of features that flinched.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+
+import pandas as pd
+
+from .perturb import delay
+from .report import BYPASS, LEAK, OK, Finding, Report
+
+__all__ = ["check"]
+
+Sources = Mapping[str, pd.DataFrame]
+Compute = Callable[[Sources], pd.DataFrame]
+
+
+def _as_frame(result: object, label: str) -> pd.DataFrame:
+    if isinstance(result, pd.Series):
+        return result.to_frame()
+    if isinstance(result, pd.DataFrame):
+        return result
+    raise TypeError(f"{label} must return a DataFrame or Series, got {type(result).__name__}")
+
+
+def _changed(left: pd.Series, right: pd.Series, tolerance: float) -> tuple[bool, float]:
+    """Did this column move? NaN in the same place counts as unchanged."""
+    both_null = left.isna() & right.isna()
+    if (left.isna() != right.isna()).any():
+        return True, float("inf")
+    if pd.api.types.is_numeric_dtype(left) and pd.api.types.is_numeric_dtype(right):
+        gap = (left - right).abs()
+        gap = gap[~both_null]
+        worst = float(gap.max()) if len(gap) else 0.0
+        return worst > tolerance, worst
+    unequal = (left != right) & ~both_null
+    return bool(unequal.any()), float(unequal.sum())
+
+
+def check(
+    compute: Compute,
+    sources: Sources,
+    timestamps: Mapping[str, str],
+    declared: Mapping[str, Sequence[str]],
+    *,
+    by: pd.Timedelta = pd.Timedelta(days=30),
+    tolerance: float = 0.0,
+    perturb: Callable[[pd.DataFrame, str, pd.Timedelta], pd.DataFrame] = delay,
+) -> Report:
+    """Perturb each source's availability in turn; report features that moved.
+
+    Args:
+        compute: builds your features. Takes the sources dict, returns a frame
+            whose columns are features. Must be deterministic -- this is checked.
+        sources: name -> raw table. Each carries an availability timestamp.
+        timestamps: source name -> the column holding that timestamp, or None for
+            a static table with no clock, which is skipped.
+        declared: feature name -> the sources it is supposed to read. A feature
+            absent from this mapping is treated as declaring nothing, which is
+            the strictest reading and usually what you want for a new pipeline.
+        by: how far to move availability. Large enough to bite, small enough to
+            leave rows in the window.
+        tolerance: treat changes at or below this as no change. Leave at 0.0
+            unless your pipeline has genuine floating-point nondeterminism, and
+            prefer to fix that instead.
+        perturb: the perturbation. `delay` removes information and is the safe
+            default; see `leakprobe.perturb` for others.
+
+    Returns:
+        A Report. `report.leaks` is the list of undeclared dependencies, and
+        `report.raise_for_leaks()` turns them into a test failure.
+    """
+    absent = set(sources) - set(timestamps)
+    if absent:
+        raise KeyError(
+            f"No availability column given for source(s): {sorted(absent)}. "
+            "Pass None for a static table that has no clock."
+        )
+
+    unknown = {s for deps in declared.values() for s in deps} - set(sources)
+    if unknown:
+        raise KeyError(f"declared refers to unknown source(s): {sorted(unknown)}")
+
+    baseline = _as_frame(compute(sources), "compute")
+    control = _as_frame(compute(sources), "compute")
+
+    notes: list[str] = []
+    if list(baseline.columns) != list(control.columns):
+        raise RuntimeError(
+            "compute() returned different columns on two identical calls. The test "
+            "cannot separate a leak from nondeterminism until that is fixed."
+        )
+    unstable = [c for c in baseline.columns if _changed(baseline[c], control[c], tolerance)[0]]
+    if unstable:
+        raise RuntimeError(
+            "compute() is not deterministic: "
+            + ", ".join(unstable[:5])
+            + (" ..." if len(unstable) > 5 else "")
+            + ". Seed it, or every result below is noise."
+        )
+
+    features = [str(c) for c in baseline.columns]
+    findings: list[Finding] = []
+
+    for source in sources:
+        if timestamps[source] is None:
+            notes.append(
+                f"note: {source!r} has no availability column, so nothing about it was "
+                f"perturbed. Dependencies on it are untested."
+            )
+            continue
+        moved_frame = _as_frame(
+            compute({**sources, source: perturb(sources[source], timestamps[source], by)}),
+            "compute",
+        )
+        if list(moved_frame.columns) != list(baseline.columns):
+            notes.append(
+                f"note: perturbing {source!r} changed the feature set itself, which is "
+                f"a stronger dependency than this test is designed to describe."
+            )
+            continue
+
+        for column, feature in zip(baseline.columns, features):
+            moved, worst = _changed(baseline[column], moved_frame[column], tolerance)
+            is_declared = source in declared.get(feature, ())
+            if moved and not is_declared:
+                kind = LEAK
+            elif is_declared and not moved:
+                kind = BYPASS
+            else:
+                kind = OK
+            findings.append(
+                Finding(
+                    feature=feature,
+                    source=source,
+                    kind=kind,
+                    declared=is_declared,
+                    moved=moved,
+                    max_abs_change=worst,
+                )
+            )
+
+    return Report(
+        findings=findings, features=features, sources=list(sources), notes=notes
+    )

leakprobe-0.1.0/src/leakprobe/perturb.py

@@ -0,0 +1,64 @@
+"""Ways to move when a source becomes knowable.
+
+A perturbation changes *availability*, never values. That distinction is what
+makes the test sound: if a feature's number changes after a perturbation, the
+feature read something whose timing moved, and timing is the only thing that
+moved.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+__all__ = ["delay", "advance", "use_column"]
+
+
+def _check(frame: pd.DataFrame, column: str) -> None:
+    if column not in frame.columns:
+        raise KeyError(
+            f"No availability column {column!r}. Columns present: {list(frame.columns)}"
+        )
+    if not pd.api.types.is_datetime64_any_dtype(frame[column]):
+        raise TypeError(
+            f"Availability column {column!r} is {frame[column].dtype}, not a datetime. "
+            "Parse it with pd.to_datetime first."
+        )
+
+
+def delay(frame: pd.DataFrame, column: str, by: pd.Timedelta) -> pd.DataFrame:
+    """The same rows, knowable `by` later.
+
+    The safe default. Delaying a source can only remove information, so a
+    feature that legitimately reads it will change and one that does not cannot.
+    """
+    _check(frame, column)
+    out = frame.copy()
+    out[column] = out[column] + by
+    return out
+
+
+def advance(frame: pd.DataFrame, column: str, by: pd.Timedelta) -> pd.DataFrame:
+    """The same rows, knowable `by` earlier -- look-ahead, injected on purpose.
+
+    Use when you want to measure what a leak would be *worth* rather than
+    whether one exists.
+    """
+    _check(frame, column)
+    out = frame.copy()
+    out[column] = out[column] - by
+    return out
+
+
+def use_column(frame: pd.DataFrame, column: str, source: str) -> pd.DataFrame:
+    """Availability taken from another column.
+
+    Models the specific mistake of treating a record as knowable when the period
+    it describes ended, rather than when it was published: `use_column(facts,
+    "available_at", "period_end")`.
+    """
+    _check(frame, column)
+    if source not in frame.columns:
+        raise KeyError(f"No column {source!r} to take availability from.")
+    out = frame.copy()
+    out[column] = out[source]
+    return out

leakprobe-0.1.0/src/leakprobe/report.py

@@ -0,0 +1,109 @@
+"""The verdict, and how it prints."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+__all__ = ["Finding", "Report"]
+
+LEAK = "leak"
+BYPASS = "bypass"
+OK = "ok"
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One (feature, source) pair, and what the perturbation did to it."""
+
+    feature: str
+    source: str
+    kind: str
+    declared: bool
+    moved: bool
+    max_abs_change: float
+
+    def __str__(self) -> str:
+        if self.kind == LEAK:
+            return (
+                f"{self.feature} moved when {self.source} was perturbed, and does not "
+                f"declare it (max change {self.max_abs_change:.3g})"
+            )
+        if self.kind == BYPASS:
+            return (
+                f"{self.feature} declares {self.source} but did not move when its clock "
+                f"did -- it reads the source without consulting availability, or the "
+                f"declaration is stale"
+            )
+        return f"{self.feature} / {self.source}: as declared"
+
+
+@dataclass
+class Report:
+    findings: list[Finding] = field(default_factory=list)
+    features: list[str] = field(default_factory=list)
+    sources: list[str] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def leaks(self) -> list[Finding]:
+        """Undeclared dependencies. These are the bugs."""
+        return [f for f in self.findings if f.kind == LEAK]
+
+    @property
+    def bypassed(self) -> list[Finding]:
+        """Declared dependencies that did not respond to their own clock.
+
+        Either the declaration is stale, or the feature reaches the source by a
+        path that ignores its availability column -- which is how look-ahead
+        usually gets in. Not a failure on its own; worth reading.
+        """
+        return [f for f in self.findings if f.kind == BYPASS]
+
+    @property
+    def clean(self) -> bool:
+        return not self.leaks
+
+    def raise_for_leaks(self) -> None:
+        """For use in a test suite: turn a leak into a failure."""
+        if self.leaks:
+            listed = "\n".join(f"  - {f}" for f in self.leaks)
+            raise AssertionError(f"{len(self.leaks)} undeclared dependencies:\n{listed}")
+
+    def __str__(self) -> str:
+        width = max((len(f) for f in self.features), default=7)
+        lines = [
+            f"{len(self.features)} features x {len(self.sources)} sources",
+            "",
+            f"{'feature':<{width}}  " + "  ".join(f"{s:>12}" for s in self.sources),
+        ]
+        by_pair = {(f.feature, f.source): f for f in self.findings}
+        for feature in self.features:
+            cells = []
+            for source in self.sources:
+                found = by_pair.get((feature, source))
+                if found is None:
+                    cells.append(f"{'-':>12}")
+                elif found.kind == LEAK:
+                    cells.append(f"{'LEAK':>12}")
+                elif found.kind == BYPASS:
+                    cells.append(f"{'bypass?':>12}")
+                elif found.declared:
+                    cells.append(f"{'reads it':>12}")
+                else:
+                    cells.append(f"{'exactly 0':>12}")
+            lines.append(f"{feature:<{width}}  " + "  ".join(cells))
+
+        lines.append("")
+        if self.leaks:
+            lines.append(f"{len(self.leaks)} undeclared dependencies:")
+            lines += [f"  - {f}" for f in self.leaks]
+        else:
+            lines.append("No undeclared dependencies.")
+        if self.bypassed:
+            lines.append("")
+            lines.append("Declared, but did not respond to the source's clock:")
+            lines += [f"  - {f}" for f in self.bypassed]
+        if self.notes:
+            lines.append("")
+            lines += self.notes
+        return "\n".join(lines)

leakprobe-0.1.0/src/leakprobe.egg-info/PKG-INFO

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: leakprobe
+Version: 0.1.0
+Summary: Find features that read data they were never supposed to see. Temporal leakage detection by metamorphic testing.
+Author: Raunak Sood
+License: MIT
+Project-URL: Homepage, https://github.com/quantraunak/leakprobe
+Project-URL: Issues, https://github.com/quantraunak/leakprobe/issues
+Keywords: leakage,data-leakage,machine-learning,testing,time-series,point-in-time,feature-engineering
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: numpy>=1.26; extra == "dev"
+Provides-Extra: examples
+Requires-Dist: numpy>=1.26; extra == "examples"
+Requires-Dist: openpyxl>=3.1; extra == "examples"
+Dynamic: license-file
+
+# leakprobe
+
+[![CI](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml/badge.svg)](https://github.com/quantraunak/leakprobe/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/leakprobe.svg)](https://pypi.org/project/leakprobe/) [![Python](https://img.shields.io/pypi/pyversions/leakprobe.svg)](https://pypi.org/project/leakprobe/)
+
+
+**Find features that read data they were never supposed to see.**
+
+Temporal leakage is the most expensive quiet bug in applied ML. A feature reads
+something that wasn't knowable yet, nothing throws, no number looks implausible,
+and your model gets better. You find out in production, if you find out at all.
+
+`leakprobe` finds it without needing to know the right answer. It needs one
+thing you already know: **a change to your data that your features must be
+invariant to.**
+
+```bash
+pip install leakprobe
+```
+
+```python
+import leakprobe as lp
+
+report = lp.check(
+    compute=build_features,                  # (sources) -> DataFrame of features
+    sources={"events": events, "tickets": tickets},
+    timestamps={"events": "occurred_at", "tickets": "resolved_at"},
+    declared={
+        "total_spend":      ["events"],
+        "event_count":      ["events"],
+        "tickets_resolved": ["tickets"],
+        "avg_severity":     ["tickets"],
+    },
+)
+report.raise_for_leaks()      # fails your test suite if anything leaked
+```
+
+```
+4 features x 2 sources
+
+feature                 events       tickets
+total_spend           reads it     exactly 0
+event_count           reads it     exactly 0
+tickets_resolved     exactly 0      reads it
+avg_severity         exactly 0       bypass?
+
+No undeclared dependencies.
+
+Declared, but did not respond to the source's clock:
+  - avg_severity declares tickets but did not move when its clock did -- it
+    reads the source without consulting availability, or the declaration is stale
+```
+
+That last line is a real bug. `avg_severity` filters tickets on `opened_at`
+instead of `resolved_at`, so tickets that were still open at scoring time leak
+in — exactly the ones that predict churn. Nothing about the code looks wrong.
+
+## Try it on real data
+
+```bash
+pip install leakprobe openpyxl
+python examples/online_retail.py
+```
+
+The UCI Online Retail set: 397,924 real orders and 8,905 returns across 4,372
+customers of a UK gift retailer, 2010-2011. Six ordinary per-customer features
+built as of a cutoff, two of them wrong in the two ways temporal leakage
+actually happens. Neither raises. Both are caught:
+
+```
+feature               orders       returns
+total_spend         reads it     exactly 0
+order_count         reads it     exactly 0
+recency_days        reads it     exactly 0
+return_count       exactly 0      reads it
+avg_unit_price       bypass?     exactly 0
+net_spend           reads it          LEAK
+
+1 undeclared dependencies:
+  - net_spend moved when returns was perturbed, and does not declare it (max change 7.46e+03)
+
+Declared, but did not respond to the source's clock:
+  - avg_unit_price declares orders but did not move when its clock did
+```
+
+`avg_unit_price` is missing one cutoff filter, so it averages invoices that had
+not happened yet. `net_spend` reaches into the returns table without declaring
+it, silently inheriting that table's latency. One dropped subscript and one
+undeclared read -- the two shapes this bug takes in production.
+
+## How it works
+
+Four steps, and no ground truth anywhere in them.
+
+1. **Move when a source became knowable.** Not its values — only its
+   availability timestamp. `delay` pushes it later, which can only ever remove
+   information.
+2. **Recompute every feature.**
+3. **Compare, exactly.** A feature that genuinely cannot read that source gets
+   the identical input arrays through the identical code and returns bit-identical
+   floats. Its difference is `0.0`, not `1e-15`. So any movement at all is proof
+   of a dependency, not a number you have to squint at.
+4. **Check what moved against what you declared.** Anything in one list and not
+   the other is the finding.
+
+Two kinds of finding:
+
+- **`leak`** — moved, but doesn't declare the source. It has a dependency you
+  didn't know about. In a temporal pipeline, an unknown dependency on *when*
+  data arrived is look-ahead.
+- **`bypass?`** — declares the source but didn't move when that source's clock
+  did. It's reaching the data by a path that ignores availability, which is how
+  look-ahead usually gets in. Not a failure on its own. Worth reading.
+
+Before any of that, `check` runs `compute` twice on untouched inputs and refuses
+to continue if the two runs disagree. A nondeterministic pipeline makes every
+result below it noise, so that's a hard error rather than a warning.
+
+## Declaring nothing
+
+You don't have to write the `declared` map. Leave it out and every real timing
+dependency is reported:
+
+```python
+report = lp.check(compute, sources, timestamps, declared={})
+for f in report.leaks:
+    print(f.feature, "reads", f.source)
+```
+
+That's the fastest way to answer "what does this pipeline actually depend on"
+for code you inherited, which is usually a shorter list than the author believed.
+
+## In CI
+
+```python
+def test_no_temporal_leakage():
+    lp.check(build_features, SOURCES, TIMESTAMPS, DECLARED).raise_for_leaks()
+```
+
+The declaration map becomes the thing code review argues about, which is where
+that argument belongs.
+
+## Perturbations
+
+| | |
+|---|---|
+| `delay(frame, col, by)` | knowable later. The safe default: removes information only. |
+| `advance(frame, col, by)` | knowable earlier. Injects look-ahead on purpose, to measure what a leak is worth. |
+| `use_column(frame, col, other)` | availability taken from another column. Models "treated as knowable when the period ended, not when it was published." |
+
+```python
+report = lp.check(..., perturb=lp.advance, by=pd.Timedelta(days=90))
+```
+
+## What it will not catch
+
+**Dependencies that don't flow through a timestamp.** The perturbation moves
+availability, so it reveals as-of joins, merges on a date, and windows anchored
+to one. A feature that reads a source's values with no reference to when they
+arrived is invariant to it and will not be flagged. In practice this costs less
+than it sounds: look-ahead *is* a dependency on timing, so the leaks worth
+catching are the detectable ones. The boundary is asserted in the test suite so
+it can't quietly stop being true.
+
+**Sources with no clock.** Pass `None` and static tables are skipped, with a
+note saying dependencies on them went untested.
+
+**Leakage across rows rather than time** — target encoding fit on the full
+dataset, a scaler fit before the split. Different bug, different tool.
+
+## Where this came from
+
+The technique is metamorphic testing, which software testing has used for
+decades and research code almost never does. This is an extraction of a check
+built for a quantitative finance pipeline, where the sources are stock prices
+and regulatory filings and the question is whether a factor read an earnings
+figure before it was published.
+
+It caught a real one. A factor called `turnover_1m` was classified as
+price-and-volume — it's built from volume, it lives in `price.py`, and every
+human who looked at it filed it under prices. It divides by shares outstanding,
+which comes off a filing. It was the only member of its group that moved when
+the filing calendar shifted, while eleven genuine price factors held at exactly
+zero. That measurement is written up in
+[bias-fingerprints](https://github.com/quantraunak/bias-fingerprints).
+
+## Install
+
+```bash
+pip install leakprobe          # pandas is the only dependency
+```
+
+Python 3.10+.
+
+## Licence
+
+MIT.

leakprobe-0.1.0/src/leakprobe.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+src/leakprobe/__init__.py
+src/leakprobe/core.py
+src/leakprobe/perturb.py
+src/leakprobe/report.py
+src/leakprobe.egg-info/PKG-INFO
+src/leakprobe.egg-info/SOURCES.txt
+src/leakprobe.egg-info/dependency_links.txt
+src/leakprobe.egg-info/requires.txt
+src/leakprobe.egg-info/top_level.txt
+tests/test_core.py
+tests/test_perturb.py

leakprobe-0.1.0/src/leakprobe.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

leakprobe-0.1.0/src/leakprobe.egg-info/requires.txt

@@ -0,0 +1,9 @@
+pandas>=2.0
+
+[dev]
+pytest>=7.4
+numpy>=1.26
+
+[examples]
+numpy>=1.26
+openpyxl>=3.1

leakprobe-0.1.0/src/leakprobe.egg-info/top_level.txt

@@ -0,0 +1 @@
+leakprobe

leakprobe-0.1.0/tests/test_core.py

@@ -0,0 +1,178 @@
+"""The claims the tool rests on, each broken on purpose."""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import leakprobe as lc
+
+
+@pytest.fixture
+def sources():
+    days = pd.date_range("2024-01-01", periods=120, freq="D")
+    rng = np.random.default_rng(0)
+    prices = pd.DataFrame(
+        {
+            "available_at": days,
+            "price": 100 + rng.normal(0, 1, len(days)).cumsum(),
+            "volume": rng.integers(1_000, 5_000, len(days)),
+        }
+    )
+    quarters = pd.to_datetime(["2023-12-31", "2024-03-31"])
+    filings = pd.DataFrame(
+        {
+            "period_end": quarters,
+            "available_at": quarters + pd.Timedelta(days=34),
+            "earnings": [4.0, 5.0],
+            "shares": [1_000.0, 1_100.0],
+        }
+    )
+    return {"prices": prices, "filings": filings}
+
+
+def _as_of(filings, dates, column):
+    """Latest value knowable on each date. The whole point is that this respects
+    `available_at`, so moving it moves the answer."""
+    ordered = filings.sort_values("available_at")
+    idx = np.searchsorted(ordered["available_at"].to_numpy(), dates.to_numpy(), side="right") - 1
+    out = np.where(idx >= 0, ordered[column].to_numpy()[np.clip(idx, 0, None)], np.nan)
+    return pd.Series(out, index=range(len(dates)))
+
+
+def build(sources):
+    prices, filings = sources["prices"], sources["filings"]
+    dates = prices["available_at"]
+    return pd.DataFrame(
+        {
+            "momentum": prices["price"].pct_change(20),
+            "earnings_yield": _as_of(filings, dates, "earnings") / prices["price"],
+            # Looks like a price feature. Divides by a filed number.
+            "turnover": prices["volume"] / _as_of(filings, dates, "shares"),
+        }
+    )
+
+
+def test_price_only_feature_is_exactly_unchanged(sources):
+    report = lc.check(
+        build, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={"momentum": ["prices"], "earnings_yield": ["filings", "prices"],
+                  "turnover": ["prices", "filings"]},
+    )
+    hit = next(f for f in report.findings if f.feature == "momentum" and f.source == "filings")
+    assert not hit.moved
+    assert hit.max_abs_change == 0.0, "not approximately zero -- exactly zero"
+    assert report.clean
+
+
+def test_undeclared_filing_dependency_is_caught(sources):
+    """turnover divides volume by shares outstanding, so it reads a filing.
+    Declared as price-only, which is the mistake this tool exists to catch."""
+    report = lc.check(
+        build, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={"momentum": ["prices"], "earnings_yield": ["filings", "prices"],
+                  "turnover": ["prices"]},
+    )
+    assert [f.feature for f in report.leaks] == ["turnover"]
+    assert report.leaks[0].source == "filings"
+    assert not report.clean
+    with pytest.raises(AssertionError, match="turnover"):
+        report.raise_for_leaks()
+
+
+def test_declaring_nothing_flags_every_timing_dependency(sources):
+    """Everything that consults a timestamp is caught. `momentum` is not, and
+    that is the method's boundary rather than a miss: it reads price *values*
+    and never asks when they arrived, so shifting availability uniformly cannot
+    move it. See test_values_read_without_consulting_time_are_invisible."""
+    report = lc.check(
+        build, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={},
+    )
+    assert {f.feature for f in report.leaks} == {"earnings_yield", "turnover"}
+
+
+def test_declared_but_clock_insensitive_is_reported_separately(sources):
+    report = lc.check(
+        build, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={"momentum": ["prices", "filings"], "earnings_yield": ["filings", "prices"],
+                  "turnover": ["prices", "filings"]},
+    )
+    assert set((f.feature, f.source) for f in report.bypassed) == {
+        ("momentum", "filings"),
+        ("momentum", "prices"),
+    }
+    assert report.clean, "a stale declaration is not a leak"
+
+
+def test_nondeterministic_compute_is_refused(sources):
+    def unstable(src):
+        return pd.DataFrame({"x": np.random.default_rng().normal(size=len(src["prices"]))})
+
+    with pytest.raises(RuntimeError, match="not deterministic"):
+        lc.check(unstable, sources,
+                 timestamps={"prices": "available_at", "filings": "available_at"},
+                 declared={})
+
+
+def test_unknown_source_in_declared_is_refused(sources):
+    with pytest.raises(KeyError, match="unknown source"):
+        lc.check(build, sources,
+                 timestamps={"prices": "available_at", "filings": "available_at"},
+                 declared={"momentum": ["typo_source"]})
+
+
+def test_missing_timestamp_column_is_refused(sources):
+    with pytest.raises(KeyError, match="availability column"):
+        lc.check(build, sources, timestamps={"prices": "available_at"}, declared={})
+
+
+def test_nan_in_the_same_place_is_not_a_change(sources):
+    """A feature that is NaN for its first 20 rows in both runs has not moved."""
+    report = lc.check(
+        build, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={"momentum": ["prices"], "earnings_yield": ["filings", "prices"],
+                  "turnover": ["prices", "filings"]},
+    )
+    assert build(sources)["momentum"].isna().sum() > 0
+    assert report.clean
+
+
+def test_values_read_without_consulting_time_are_invisible(sources):
+    """The limitation, asserted so it cannot quietly stop being true.
+
+    The perturbation moves availability, so it can only reveal dependencies that
+    flow through a timestamp -- an as-of join, a merge on a date, a window
+    anchored to one. A feature that reads a source's values directly, with no
+    reference to when they arrived, is invariant to it and will not be flagged.
+
+    That boundary is not much of a loss in practice: look-ahead is by definition
+    a dependency on timing, so the leaks worth catching are the detectable ones.
+    """
+    def timeless(src):
+        # Reads filings values, never their availability. Undetectable here.
+        return pd.DataFrame({"mean_earnings": [src["filings"]["earnings"].mean()]})
+
+    report = lc.check(
+        timeless, sources,
+        timestamps={"prices": "available_at", "filings": "available_at"},
+        declared={},
+    )
+    assert report.clean
+    assert report.leaks == []
+
+
+def test_source_without_a_clock_is_skipped_and_said_so(sources):
+    static = pd.DataFrame({"region": ["us", "eu"]})
+    report = lc.check(
+        lambda src: pd.DataFrame({"x": [len(src["static"])]}),
+        {**sources, "static": static},
+        timestamps={"prices": "available_at", "filings": "available_at", "static": None},
+        declared={},
+    )
+    assert any("no availability column" in n for n in report.notes)
+    assert all(f.source != "static" for f in report.findings)

leakprobe-0.1.0/tests/test_perturb.py

@@ -0,0 +1,38 @@
+import pandas as pd
+import pytest
+
+from leakprobe import advance, delay, use_column
+
+
+@pytest.fixture
+def facts():
+    ends = pd.to_datetime(["2024-03-31", "2024-06-30"])
+    return pd.DataFrame({"period_end": ends, "available_at": ends + pd.Timedelta(days=34)})
+
+
+def test_delay_moves_availability_only(facts):
+    out = delay(facts, "available_at", pd.Timedelta(days=30))
+    assert (out["available_at"] - facts["available_at"] == pd.Timedelta(days=30)).all()
+    pd.testing.assert_series_equal(out["period_end"], facts["period_end"])
+
+
+def test_advance_is_the_inverse(facts):
+    by = pd.Timedelta(days=7)
+    pd.testing.assert_frame_equal(advance(delay(facts, "available_at", by), "available_at", by), facts)
+
+
+def test_use_column_models_the_period_end_join(facts):
+    out = use_column(facts, "available_at", "period_end")
+    pd.testing.assert_series_equal(out["available_at"], facts["period_end"], check_names=False)
+
+
+def test_input_is_never_mutated(facts):
+    before = facts.copy()
+    delay(facts, "available_at", pd.Timedelta(days=1))
+    pd.testing.assert_frame_equal(facts, before)
+
+
+def test_non_datetime_column_is_refused(facts):
+    bad = facts.assign(available_at=[1, 2])
+    with pytest.raises(TypeError, match="not a datetime"):
+        delay(bad, "available_at", pd.Timedelta(days=1))