leaklens-ml 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leaklens_ml-0.4.0/LICENSE +21 -0
- leaklens_ml-0.4.0/PKG-INFO +136 -0
- leaklens_ml-0.4.0/README.md +102 -0
- leaklens_ml-0.4.0/leaklens/__init__.py +1 -0
- leaklens_ml-0.4.0/leaklens/__main__.py +3 -0
- leaklens_ml-0.4.0/leaklens/api.py +77 -0
- leaklens_ml-0.4.0/leaklens/arena.py +140 -0
- leaklens_ml-0.4.0/leaklens/audit.py +18 -0
- leaklens_ml-0.4.0/leaklens/cli.py +75 -0
- leaklens_ml-0.4.0/leaklens/leakage.py +118 -0
- leaklens_ml-0.4.0/leaklens/scanner.py +114 -0
- leaklens_ml-0.4.0/leaklens/ui.py +71 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/PKG-INFO +136 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/SOURCES.txt +25 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/dependency_links.txt +1 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/entry_points.txt +2 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/requires.txt +19 -0
- leaklens_ml-0.4.0/leaklens_ml.egg-info/top_level.txt +1 -0
- leaklens_ml-0.4.0/pyproject.toml +36 -0
- leaklens_ml-0.4.0/setup.cfg +4 -0
- leaklens_ml-0.4.0/tests/test_api.py +60 -0
- leaklens_ml-0.4.0/tests/test_arena.py +43 -0
- leaklens_ml-0.4.0/tests/test_badge.py +12 -0
- leaklens_ml-0.4.0/tests/test_group.py +29 -0
- leaklens_ml-0.4.0/tests/test_leakage.py +49 -0
- leaklens_ml-0.4.0/tests/test_multi_scan.py +16 -0
- leaklens_ml-0.4.0/tests/test_scanner.py +73 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Eyad
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: leaklens-ml
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Audit datasets and ML pipelines for data leakage
|
|
5
|
+
Author: Eyad
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/eyadstein/leaklens
|
|
8
|
+
Project-URL: Issues, https://github.com/eyadstein/leaklens/issues
|
|
9
|
+
Keywords: data-leakage,machine-learning,mlops,data-science,scikit-learn,static-analysis
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: pandas>=2.0
|
|
18
|
+
Requires-Dist: numpy>=1.24
|
|
19
|
+
Requires-Dist: scikit-learn>=1.3
|
|
20
|
+
Provides-Extra: arena
|
|
21
|
+
Requires-Dist: mlflow>=2.9; extra == "arena"
|
|
22
|
+
Provides-Extra: app
|
|
23
|
+
Requires-Dist: fastapi>=0.110; extra == "app"
|
|
24
|
+
Requires-Dist: uvicorn>=0.27; extra == "app"
|
|
25
|
+
Requires-Dist: python-multipart>=0.0.9; extra == "app"
|
|
26
|
+
Requires-Dist: streamlit>=1.30; extra == "app"
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
29
|
+
Requires-Dist: mlflow>=2.9; extra == "dev"
|
|
30
|
+
Requires-Dist: fastapi>=0.110; extra == "dev"
|
|
31
|
+
Requires-Dist: httpx>=0.26; extra == "dev"
|
|
32
|
+
Requires-Dist: python-multipart>=0.0.9; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# LeakLens
|
|
36
|
+
|
|
37
|
+
[](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml)
|
|
38
|
+
[](#trust-score-badge)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
|
|
41
|
+
**Catch data leakage before it fools you.** Most "99% accuracy" models are leaking. LeakLens audits both your **data** and your **code**, then proves what survives by training three model families on a leak-free pipeline.
|
|
42
|
+
|
|
43
|
+
## What it catches
|
|
44
|
+
| Where | Leak |
|
|
45
|
+
|---|---|
|
|
46
|
+
| Data | Target leakage (a single feature that almost perfectly predicts the label), ID-like columns, train/test row overlap, temporal leakage, entity/group overlap (`--group COL`) |
|
|
47
|
+
| Code (`.py`, `.ipynb`) | Scaler/encoder/imputer fitted before the split, SMOTE before the split, transformers fitted on test/validation data |
|
|
48
|
+
|
|
49
|
+
Every audit returns a 0-100 trust score. The CLI exits with code 1 on a high-severity leak, so it gates CI.
|
|
50
|
+
|
|
51
|
+
## Demo
|
|
52
|
+
|
|
53
|
+
$ leaklens demo.csv --target is_fraud --arena
|
|
54
|
+
LeakLens trust score: 50/100
|
|
55
|
+
[HIGH] target_leakage :: chargeback_flag :: alone predicts target (roc_auc=1.000)
|
|
56
|
+
[MEDIUM] id_like :: txn_id :: 100.0% unique values; likely an identifier
|
|
57
|
+
|
|
58
|
+
Arena ranked by pr_auc (auto-dropped: chargeback_flag, txn_id)
|
|
59
|
+
model roc_auc pr_auc best_f1
|
|
60
|
+
neural_net 0.8374 0.4219 0.4353
|
|
61
|
+
linear 0.8410 0.4215 0.4304
|
|
62
|
+
tree_ensemble 0.8242 0.3562 0.4103
|
|
63
|
+
|
|
64
|
+
A leaked column makes accuracy look perfect. LeakLens drops it and shows the honest numbers.
|
|
65
|
+
|
|
66
|
+
## Quick start
|
|
67
|
+
|
|
68
|
+
pip install -e “.[dev,app]”
|
|
69
|
+
leaklens train.csv --target label --test test.csv --time date
|
|
70
|
+
leaklens --scan notebooks/
|
|
71
|
+
leaklens train.csv --target label --test test.csv --group patient_id
|
|
72
|
+
python examples/make_demo.py
|
|
73
|
+
leaklens demo.csv --target is_fraud --arena
|
|
74
|
+
mlflow ui --backend-store-uri sqlite:///mlflow.db
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
## Model arena
|
|
78
|
+
Auto-drops leaking and ID columns, then trains linear, gradient-boosted tree and neural-net models inside leak-safe pipelines (imputation, scaling and encoding are fitted on training data only). Runs are tracked in local MLflow (SQLite) and the winner is saved to `models/best.joblib`. Add `--time COL` for a temporal split.
|
|
79
|
+
|
|
80
|
+
## API and UI
|
|
81
|
+
|
|
82
|
+
uvicorn leaklens.api:app --port 8000 # docs at /docs
|
|
83
|
+
streamlit run leaklens/ui.py
|
|
84
|
+
|
|
85
|
+
Endpoints: `GET /health`, `POST /audit`, `POST /scan`, `POST /predict`.
|
|
86
|
+
|
|
87
|
+
## Docker
|
|
88
|
+
|
|
89
|
+
docker compose up --build
|
|
90
|
+
|
|
91
|
+
API on http://localhost:8000/docs, UI on http://localhost:8501.
|
|
92
|
+
|
|
93
|
+
## GitHub Action
|
|
94
|
+
```yaml
|
|
95
|
+
- uses: actions/checkout@v4
|
|
96
|
+
- uses: eyadstein/leaklens@main
|
|
97
|
+
with:
|
|
98
|
+
path: notebooks
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Pre-commit hook
|
|
102
|
+
```yaml
|
|
103
|
+
repos:
|
|
104
|
+
- repo: https://github.com/eyadstein/leaklens
|
|
105
|
+
rev: main
|
|
106
|
+
hooks:
|
|
107
|
+
- id: leaklens
|
|
108
|
+
```
|
|
109
|
+
Blocks a commit when a notebook or script fits a transformer before the split.
|
|
110
|
+
|
|
111
|
+
## Trust score badge
|
|
112
|
+
|
|
113
|
+
leaklens --scan . --badge badge.json
|
|
114
|
+
|
|
115
|
+
Commit `badge.json`, then point a shields.io endpoint badge at its raw URL (as at the top of this file).
|
|
116
|
+
|
|
117
|
+
## Architecture
|
|
118
|
+
```mermaid
|
|
119
|
+
flowchart LR
|
|
120
|
+
A[CSV / notebooks] --> B[Data audit + leakage detectors]
|
|
121
|
+
A --> C[Static code scanner]
|
|
122
|
+
B --> D[Trust score]
|
|
123
|
+
C --> D
|
|
124
|
+
B --> E[Model arena + MLflow]
|
|
125
|
+
E --> F[best.joblib]
|
|
126
|
+
F --> G[FastAPI /predict]
|
|
127
|
+
D --> H[CLI / Streamlit / GitHub Action]
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Roadmap
|
|
131
|
+
- [x] Core detectors, static scanner, model arena, API, UI
|
|
132
|
+
- [x] Docker, GitHub Action, badge, CI
|
|
133
|
+
- [x] Group/entity leakage detector
|
|
134
|
+
- [x] PyPI packaging + release workflow (PyPI name `leaklens-ml`, command stays `leaklens`; create a GitHub release to publish)
|
|
135
|
+
|
|
136
|
+
MIT licensed.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# LeakLens
|
|
2
|
+
|
|
3
|
+
[](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml)
|
|
4
|
+
[](#trust-score-badge)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
**Catch data leakage before it fools you.** Most "99% accuracy" models are leaking. LeakLens audits both your **data** and your **code**, then proves what survives by training three model families on a leak-free pipeline.
|
|
8
|
+
|
|
9
|
+
## What it catches
|
|
10
|
+
| Where | Leak |
|
|
11
|
+
|---|---|
|
|
12
|
+
| Data | Target leakage (a single feature that almost perfectly predicts the label), ID-like columns, train/test row overlap, temporal leakage, entity/group overlap (`--group COL`) |
|
|
13
|
+
| Code (`.py`, `.ipynb`) | Scaler/encoder/imputer fitted before the split, SMOTE before the split, transformers fitted on test/validation data |
|
|
14
|
+
|
|
15
|
+
Every audit returns a 0-100 trust score. The CLI exits with code 1 on a high-severity leak, so it gates CI.
|
|
16
|
+
|
|
17
|
+
## Demo
|
|
18
|
+
|
|
19
|
+
$ leaklens demo.csv --target is_fraud --arena
|
|
20
|
+
LeakLens trust score: 50/100
|
|
21
|
+
[HIGH] target_leakage :: chargeback_flag :: alone predicts target (roc_auc=1.000)
|
|
22
|
+
[MEDIUM] id_like :: txn_id :: 100.0% unique values; likely an identifier
|
|
23
|
+
|
|
24
|
+
Arena ranked by pr_auc (auto-dropped: chargeback_flag, txn_id)
|
|
25
|
+
model roc_auc pr_auc best_f1
|
|
26
|
+
neural_net 0.8374 0.4219 0.4353
|
|
27
|
+
linear 0.8410 0.4215 0.4304
|
|
28
|
+
tree_ensemble 0.8242 0.3562 0.4103
|
|
29
|
+
|
|
30
|
+
A leaked column makes accuracy look perfect. LeakLens drops it and shows the honest numbers.
|
|
31
|
+
|
|
32
|
+
## Quick start
|
|
33
|
+
|
|
34
|
+
pip install -e “.[dev,app]”
|
|
35
|
+
leaklens train.csv --target label --test test.csv --time date
|
|
36
|
+
leaklens --scan notebooks/
|
|
37
|
+
leaklens train.csv --target label --test test.csv --group patient_id
|
|
38
|
+
python examples/make_demo.py
|
|
39
|
+
leaklens demo.csv --target is_fraud --arena
|
|
40
|
+
mlflow ui --backend-store-uri sqlite:///mlflow.db
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## Model arena
|
|
44
|
+
Auto-drops leaking and ID columns, then trains linear, gradient-boosted tree and neural-net models inside leak-safe pipelines (imputation, scaling and encoding are fitted on training data only). Runs are tracked in local MLflow (SQLite) and the winner is saved to `models/best.joblib`. Add `--time COL` for a temporal split.
|
|
45
|
+
|
|
46
|
+
## API and UI
|
|
47
|
+
|
|
48
|
+
uvicorn leaklens.api:app --port 8000 # docs at /docs
|
|
49
|
+
streamlit run leaklens/ui.py
|
|
50
|
+
|
|
51
|
+
Endpoints: `GET /health`, `POST /audit`, `POST /scan`, `POST /predict`.
|
|
52
|
+
|
|
53
|
+
## Docker
|
|
54
|
+
|
|
55
|
+
docker compose up --build
|
|
56
|
+
|
|
57
|
+
API on http://localhost:8000/docs, UI on http://localhost:8501.
|
|
58
|
+
|
|
59
|
+
## GitHub Action
|
|
60
|
+
```yaml
|
|
61
|
+
- uses: actions/checkout@v4
|
|
62
|
+
- uses: eyadstein/leaklens@main
|
|
63
|
+
with:
|
|
64
|
+
path: notebooks
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Pre-commit hook
|
|
68
|
+
```yaml
|
|
69
|
+
repos:
|
|
70
|
+
- repo: https://github.com/eyadstein/leaklens
|
|
71
|
+
rev: main
|
|
72
|
+
hooks:
|
|
73
|
+
- id: leaklens
|
|
74
|
+
```
|
|
75
|
+
Blocks a commit when a notebook or script fits a transformer before the split.
|
|
76
|
+
|
|
77
|
+
## Trust score badge
|
|
78
|
+
|
|
79
|
+
leaklens --scan . --badge badge.json
|
|
80
|
+
|
|
81
|
+
Commit `badge.json`, then point a shields.io endpoint badge at its raw URL (as at the top of this file).
|
|
82
|
+
|
|
83
|
+
## Architecture
|
|
84
|
+
```mermaid
|
|
85
|
+
flowchart LR
|
|
86
|
+
A[CSV / notebooks] --> B[Data audit + leakage detectors]
|
|
87
|
+
A --> C[Static code scanner]
|
|
88
|
+
B --> D[Trust score]
|
|
89
|
+
C --> D
|
|
90
|
+
B --> E[Model arena + MLflow]
|
|
91
|
+
E --> F[best.joblib]
|
|
92
|
+
F --> G[FastAPI /predict]
|
|
93
|
+
D --> H[CLI / Streamlit / GitHub Action]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Roadmap
|
|
97
|
+
- [x] Core detectors, static scanner, model arena, API, UI
|
|
98
|
+
- [x] Docker, GitHub Action, badge, CI
|
|
99
|
+
- [x] Group/entity leakage detector
|
|
100
|
+
- [x] PyPI packaging + release workflow (PyPI name `leaklens-ml`, command stays `leaklens`; create a GitHub release to publish)
|
|
101
|
+
|
|
102
|
+
MIT licensed.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.0"
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import joblib
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .audit import audit
|
|
14
|
+
from .leakage import scan, score
|
|
15
|
+
from .scanner import scan_source
|
|
16
|
+
|
|
17
|
+
app = FastAPI(title="LeakLens", version=__version__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ScanReq(BaseModel):
|
|
21
|
+
source: str
|
|
22
|
+
name: str = "snippet.py"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PredictReq(BaseModel):
|
|
26
|
+
rows: list[dict]
|
|
27
|
+
model_path: str = "models/best.joblib"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@app.get("/health")
|
|
31
|
+
def health():
|
|
32
|
+
return {"status": "ok", "version": __version__}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.post("/audit")
|
|
36
|
+
async def audit_csv(file: UploadFile = File(...), target: str = Form(...), threshold: float = Form(0.95)):
|
|
37
|
+
try:
|
|
38
|
+
df = pd.read_csv(io.BytesIO(await file.read()))
|
|
39
|
+
except Exception as e:
|
|
40
|
+
raise HTTPException(400, f"bad CSV: {e}")
|
|
41
|
+
if target not in df.columns:
|
|
42
|
+
raise HTTPException(422, f"target '{target}' not in columns")
|
|
43
|
+
fs = scan(df, target, threshold)
|
|
44
|
+
return {"trust_score": score(fs), "audit": audit(df, target), "findings": [f.to_dict() for f in fs]}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.post("/scan")
|
|
48
|
+
def scan_code(req: ScanReq):
|
|
49
|
+
fs = scan_source(req.source, req.name)
|
|
50
|
+
return {"trust_score": score(fs), "findings": [f.to_dict() for f in fs]}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@app.post("/predict")
|
|
54
|
+
def predict(req: PredictReq):
|
|
55
|
+
p = Path(req.model_path)
|
|
56
|
+
if not p.exists():
|
|
57
|
+
raise HTTPException(404, "model not found; run: leaklens data.csv --target T --arena")
|
|
58
|
+
b = joblib.load(p)
|
|
59
|
+
df = pd.DataFrame(req.rows)
|
|
60
|
+
miss = [c for c in b["features"] if c not in df.columns]
|
|
61
|
+
if miss:
|
|
62
|
+
raise HTTPException(422, f"missing features: {miss}")
|
|
63
|
+
X = df[b["features"]].copy()
|
|
64
|
+
pipe = b["pipeline"]
|
|
65
|
+
tf = pipe.named_steps["prep"].transformers_
|
|
66
|
+
for c in tf[0][2]:
|
|
67
|
+
X[c] = pd.to_numeric(X[c], errors="coerce")
|
|
68
|
+
for c in tf[1][2]:
|
|
69
|
+
X[c] = X[c].astype(str)
|
|
70
|
+
if b["task"] == "classification":
|
|
71
|
+
out = []
|
|
72
|
+
for pr in pipe.predict_proba(X):
|
|
73
|
+
i = int(np.argmax(pr))
|
|
74
|
+
out.append({"label": b["classes"][i], "score": float(pr[1] if len(pr) == 2 else pr[i])})
|
|
75
|
+
else:
|
|
76
|
+
out = [{"value": float(v)} for v in pipe.predict(X)]
|
|
77
|
+
return {"model": b["model"], "task": b["task"], "predictions": out}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import joblib
|
|
6
|
+
import mlflow
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from sklearn.compose import ColumnTransformer
|
|
10
|
+
from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
|
|
11
|
+
from sklearn.impute import SimpleImputer
|
|
12
|
+
from sklearn.linear_model import LogisticRegression, Ridge
|
|
13
|
+
from sklearn.metrics import (accuracy_score, average_precision_score, f1_score, mean_absolute_error,
|
|
14
|
+
mean_squared_error, precision_recall_curve, precision_score, r2_score,
|
|
15
|
+
recall_score, roc_auc_score)
|
|
16
|
+
from sklearn.model_selection import train_test_split
|
|
17
|
+
from sklearn.neural_network import MLPClassifier, MLPRegressor
|
|
18
|
+
from sklearn.pipeline import Pipeline
|
|
19
|
+
from sklearn.preprocessing import OneHotEncoder, StandardScaler
|
|
20
|
+
|
|
21
|
+
from .leakage import scan
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _models(clf, seed):
|
|
25
|
+
if clf:
|
|
26
|
+
return {
|
|
27
|
+
"linear": LogisticRegression(max_iter=1000, class_weight="balanced"),
|
|
28
|
+
"tree_ensemble": HistGradientBoostingClassifier(random_state=seed),
|
|
29
|
+
"neural_net": MLPClassifier(hidden_layer_sizes=(64, 32), max_iter=300, early_stopping=True, random_state=seed),
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
"linear": Ridge(),
|
|
33
|
+
"tree_ensemble": HistGradientBoostingRegressor(random_state=seed),
|
|
34
|
+
"neural_net": MLPRegressor(hidden_layer_sizes=(64, 32), max_iter=300, early_stopping=True, random_state=seed),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _prep(num, cat):
|
|
39
|
+
return ColumnTransformer([
|
|
40
|
+
("num", Pipeline([("imp", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), num),
|
|
41
|
+
("cat", OneHotEncoder(handle_unknown="infrequent_if_exist", max_categories=20, sparse_output=False), cat),
|
|
42
|
+
])
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _metrics(clf, y, pipe, X):
|
|
46
|
+
out = {}
|
|
47
|
+
pred = pipe.predict(X)
|
|
48
|
+
if clf:
|
|
49
|
+
proba = pipe.predict_proba(X)
|
|
50
|
+
if proba.shape[1] == 2:
|
|
51
|
+
s = proba[:, 1]
|
|
52
|
+
try:
|
|
53
|
+
out["roc_auc"] = roc_auc_score(y, s)
|
|
54
|
+
out["pr_auc"] = average_precision_score(y, s)
|
|
55
|
+
p, r, _ = precision_recall_curve(y, s)
|
|
56
|
+
out["best_f1"] = float((2 * p * r / np.maximum(p + r, 1e-12)).max())
|
|
57
|
+
except ValueError:
|
|
58
|
+
pass
|
|
59
|
+
out["f1"] = f1_score(y, pred, zero_division=0)
|
|
60
|
+
out["precision"] = precision_score(y, pred, zero_division=0)
|
|
61
|
+
out["recall"] = recall_score(y, pred, zero_division=0)
|
|
62
|
+
else:
|
|
63
|
+
out["f1_macro"] = f1_score(y, pred, average="macro", zero_division=0)
|
|
64
|
+
out["accuracy"] = accuracy_score(y, pred)
|
|
65
|
+
else:
|
|
66
|
+
out["mae"] = mean_absolute_error(y, pred)
|
|
67
|
+
out["rmse"] = float(np.sqrt(mean_squared_error(y, pred)))
|
|
68
|
+
out["r2"] = r2_score(y, pred)
|
|
69
|
+
return {k: float(v) for k, v in out.items()}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def run(df, target, time_col=None, out_dir="models", tracking_uri="sqlite:///mlflow.db",
|
|
73
|
+
experiment="leaklens-arena", drop_leaks=True, seed=0):
|
|
74
|
+
d = df[df[target].notna()].copy()
|
|
75
|
+
if len(d) < 50:
|
|
76
|
+
raise ValueError("need at least 50 labeled rows")
|
|
77
|
+
dropped = []
|
|
78
|
+
if drop_leaks:
|
|
79
|
+
dropped = [f.column for f in scan(d, target)
|
|
80
|
+
if f.kind == "id_like" or (f.kind == "target_leakage" and f.severity == "high")]
|
|
81
|
+
y_raw = d[target]
|
|
82
|
+
clf = y_raw.dtype.kind not in "iuf" or y_raw.nunique() <= 20
|
|
83
|
+
if clf:
|
|
84
|
+
cats = y_raw.astype("category")
|
|
85
|
+
y = cats.cat.codes.to_numpy()
|
|
86
|
+
classes = [str(c) for c in cats.cat.categories]
|
|
87
|
+
else:
|
|
88
|
+
y = y_raw.astype(float).to_numpy()
|
|
89
|
+
classes = []
|
|
90
|
+
drop = {target, *dropped}
|
|
91
|
+
if time_col:
|
|
92
|
+
drop.add(time_col)
|
|
93
|
+
X = d.drop(columns=[c for c in drop if c in d.columns])
|
|
94
|
+
X = X.drop(columns=X.select_dtypes(include=["datetime", "datetimetz"]).columns)
|
|
95
|
+
for c in X.select_dtypes("bool").columns:
|
|
96
|
+
X[c] = X[c].astype(int)
|
|
97
|
+
num = X.select_dtypes(include=["number"]).columns.tolist()
|
|
98
|
+
cat = [c for c in X.columns if c not in num]
|
|
99
|
+
if num:
|
|
100
|
+
X[num] = X[num].replace([np.inf, -np.inf], np.nan)
|
|
101
|
+
for c in cat:
|
|
102
|
+
X[c] = X[c].astype(str)
|
|
103
|
+
n = len(d)
|
|
104
|
+
if time_col:
|
|
105
|
+
t = pd.to_datetime(d[time_col], errors="coerce").to_numpy()
|
|
106
|
+
order = np.argsort(t, kind="stable")
|
|
107
|
+
cut = int(n * 0.8)
|
|
108
|
+
tr_i, te_i = order[:cut], order[cut:]
|
|
109
|
+
else:
|
|
110
|
+
strat = y if clf and np.bincount(y).min() >= 2 else None
|
|
111
|
+
tr_i, te_i = train_test_split(np.arange(n), test_size=0.2, stratify=strat, random_state=seed)
|
|
112
|
+
if clf and len(np.unique(y[tr_i])) < 2:
|
|
113
|
+
raise ValueError("training split has a single class")
|
|
114
|
+
Xtr, Xte, ytr, yte = X.iloc[tr_i], X.iloc[te_i], y[tr_i], y[te_i]
|
|
115
|
+
primary = ("pr_auc" if len(classes) == 2 else "f1_macro") if clf else "rmse"
|
|
116
|
+
mlflow.set_tracking_uri(tracking_uri)
|
|
117
|
+
mlflow.set_experiment(experiment)
|
|
118
|
+
rows, pipes = [], {}
|
|
119
|
+
for name, model in _models(clf, seed).items():
|
|
120
|
+
pipe = Pipeline([("prep", _prep(num, cat)), ("model", model)])
|
|
121
|
+
with mlflow.start_run(run_name=name):
|
|
122
|
+
mlflow.log_params({"model": name, "task": "classification" if clf else "regression",
|
|
123
|
+
"split": "temporal" if time_col else "random", "n_train": len(tr_i),
|
|
124
|
+
"n_test": len(te_i), "dropped": ",".join(map(str, dropped)) or "none"})
|
|
125
|
+
pipe.fit(Xtr, ytr)
|
|
126
|
+
m = _metrics(clf, yte, pipe, Xte)
|
|
127
|
+
mlflow.log_metrics({k: v for k, v in m.items() if np.isfinite(v)})
|
|
128
|
+
pipes[name] = pipe
|
|
129
|
+
rows.append({"model": name, **m})
|
|
130
|
+
res = pd.DataFrame(rows)
|
|
131
|
+
if primary not in res:
|
|
132
|
+
res[primary] = np.nan
|
|
133
|
+
res = res.sort_values(primary, ascending=(primary == "rmse"), na_position="last").reset_index(drop=True)
|
|
134
|
+
Path(out_dir).mkdir(parents=True, exist_ok=True)
|
|
135
|
+
joblib.dump({"pipeline": pipes[res.loc[0, "model"]], "features": list(X.columns), "target": target,
|
|
136
|
+
"task": "classification" if clf else "regression", "classes": classes,
|
|
137
|
+
"dropped": dropped, "model": res.loc[0, "model"]}, Path(out_dir) / "best.joblib")
|
|
138
|
+
res.attrs["dropped"] = dropped
|
|
139
|
+
res.attrs["primary"] = primary
|
|
140
|
+
return res
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def audit(df: pd.DataFrame, target: str | None = None) -> dict:
|
|
7
|
+
out = {
|
|
8
|
+
"rows": int(len(df)),
|
|
9
|
+
"cols": int(df.shape[1]),
|
|
10
|
+
"duplicate_rows": int(df.duplicated().sum()),
|
|
11
|
+
"null_rate": {c: round(float(v), 4) for c, v in df.isna().mean().items() if v > 0},
|
|
12
|
+
"cardinality": {c: int(df[c].nunique(dropna=True)) for c in df.columns},
|
|
13
|
+
}
|
|
14
|
+
if target is not None and target in df.columns:
|
|
15
|
+
vc = df[target].value_counts(normalize=True)
|
|
16
|
+
out["target_distribution"] = {str(k): round(float(v), 4) for k, v in vc.items()}
|
|
17
|
+
out["minority_rate"] = round(float(vc.min()), 6)
|
|
18
|
+
return out
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from .audit import audit
|
|
9
|
+
from .leakage import group_overlap, group_risk, scan, score, split_overlap, temporal_overlap
|
|
10
|
+
from .scanner import scan_path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(argv=None) -> int:
|
|
14
|
+
p = argparse.ArgumentParser(prog="leaklens", description="Audit datasets and ML code for leakage")
|
|
15
|
+
p.add_argument("train", nargs="?")
|
|
16
|
+
p.add_argument("--target")
|
|
17
|
+
p.add_argument("--test")
|
|
18
|
+
p.add_argument("--time")
|
|
19
|
+
p.add_argument("--group", help="entity column (patient, customer...) to check for group leakage")
|
|
20
|
+
p.add_argument("--threshold", type=float, default=0.95)
|
|
21
|
+
p.add_argument("--scan", nargs="+", help="file or folder of .py/.ipynb to scan for pipeline leaks")
|
|
22
|
+
p.add_argument("--arena", action="store_true", help="compare 3 model families, tracked in MLflow")
|
|
23
|
+
p.add_argument("--out", default="models")
|
|
24
|
+
p.add_argument("--tracking-uri", default="sqlite:///mlflow.db")
|
|
25
|
+
p.add_argument("--json", action="store_true")
|
|
26
|
+
p.add_argument("--badge", help="write a shields.io endpoint JSON with the trust score")
|
|
27
|
+
a = p.parse_args(argv)
|
|
28
|
+
if not a.train and not a.scan:
|
|
29
|
+
p.error("give a CSV and/or --scan PATH")
|
|
30
|
+
if a.train and not a.target:
|
|
31
|
+
p.error("--target is required with a CSV")
|
|
32
|
+
if a.arena and not a.train:
|
|
33
|
+
p.error("--arena needs a CSV")
|
|
34
|
+
fs, rep, res = [], {}, None
|
|
35
|
+
if a.train:
|
|
36
|
+
tr = pd.read_csv(a.train)
|
|
37
|
+
fs += scan(tr, a.target, a.threshold)
|
|
38
|
+
if a.test:
|
|
39
|
+
te = pd.read_csv(a.test)
|
|
40
|
+
fs += split_overlap(tr, te)
|
|
41
|
+
if a.time:
|
|
42
|
+
fs += temporal_overlap(tr, te, a.time)
|
|
43
|
+
if a.group:
|
|
44
|
+
fs += group_risk(tr, a.group)
|
|
45
|
+
if a.test:
|
|
46
|
+
fs += group_overlap(tr, te, a.group)
|
|
47
|
+
rep["audit"] = audit(tr, a.target)
|
|
48
|
+
if a.arena:
|
|
49
|
+
try:
|
|
50
|
+
from .arena import run
|
|
51
|
+
except ImportError:
|
|
52
|
+
p.error('install extras: pip install -e ".[arena]"')
|
|
53
|
+
res = run(tr, a.target, a.time, a.out, a.tracking_uri)
|
|
54
|
+
rep["arena"] = res.round(4).to_dict("records")
|
|
55
|
+
if a.scan:
|
|
56
|
+
for s in a.scan:
|
|
57
|
+
fs += scan_path(s)
|
|
58
|
+
rep = {"trust_score": score(fs), **rep, "findings": [f.to_dict() for f in fs]}
|
|
59
|
+
if a.badge:
|
|
60
|
+
s = rep["trust_score"]
|
|
61
|
+
col = "brightgreen" if s >= 85 else "yellow" if s >= 60 else "red"
|
|
62
|
+
with open(a.badge, "w", encoding="utf-8") as fh:
|
|
63
|
+
json.dump({"schemaVersion": 1, "label": "leakage trust", "message": f"{s}/100", "color": col}, fh)
|
|
64
|
+
if a.json:
|
|
65
|
+
print(json.dumps(rep, indent=2))
|
|
66
|
+
else:
|
|
67
|
+
print(f"LeakLens trust score: {rep['trust_score']}/100")
|
|
68
|
+
for f in fs:
|
|
69
|
+
print(f"[{f.severity.upper()}] {f.kind} :: {f.column} :: {f.detail}")
|
|
70
|
+
if not fs:
|
|
71
|
+
print("No leakage found.")
|
|
72
|
+
if res is not None:
|
|
73
|
+
print(f"\nArena ranked by {res.attrs['primary']} (auto-dropped: {', '.join(map(str, res.attrs['dropped'])) or 'none'})")
|
|
74
|
+
print(res.round(4).to_string(index=False))
|
|
75
|
+
return 1 if any(f.severity == "high" for f in fs) else 0
|