leaklens-ml 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Eyad
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,136 @@
1
+ Metadata-Version: 2.4
2
+ Name: leaklens-ml
3
+ Version: 0.4.0
4
+ Summary: Audit datasets and ML pipelines for data leakage
5
+ Author: Eyad
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/eyadstein/leaklens
8
+ Project-URL: Issues, https://github.com/eyadstein/leaklens/issues
9
+ Keywords: data-leakage,machine-learning,mlops,data-science,scikit-learn,static-analysis
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Requires-Python: >=3.10
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Requires-Dist: pandas>=2.0
18
+ Requires-Dist: numpy>=1.24
19
+ Requires-Dist: scikit-learn>=1.3
20
+ Provides-Extra: arena
21
+ Requires-Dist: mlflow>=2.9; extra == "arena"
22
+ Provides-Extra: app
23
+ Requires-Dist: fastapi>=0.110; extra == "app"
24
+ Requires-Dist: uvicorn>=0.27; extra == "app"
25
+ Requires-Dist: python-multipart>=0.0.9; extra == "app"
26
+ Requires-Dist: streamlit>=1.30; extra == "app"
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest>=7; extra == "dev"
29
+ Requires-Dist: mlflow>=2.9; extra == "dev"
30
+ Requires-Dist: fastapi>=0.110; extra == "dev"
31
+ Requires-Dist: httpx>=0.26; extra == "dev"
32
+ Requires-Dist: python-multipart>=0.0.9; extra == "dev"
33
+ Dynamic: license-file
34
+
35
+ # LeakLens
36
+
37
+ [![CI](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml/badge.svg)](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml)
38
+ [![Leakage trust](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/eyadstein/leaklens/main/badge.json)](#trust-score-badge)
39
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
40
+
41
+ **Catch data leakage before it fools you.** Most "99% accuracy" models are leaking. LeakLens audits both your **data** and your **code**, then proves what survives by training three model families on a leak-free pipeline.
42
+
43
+ ## What it catches
44
+ | Where | Leak |
45
+ |---|---|
46
+ | Data | Target leakage (a single feature that almost perfectly predicts the label), ID-like columns, train/test row overlap, temporal leakage, entity/group overlap (`--group COL`) |
47
+ | Code (`.py`, `.ipynb`) | Scaler/encoder/imputer fitted before the split, SMOTE before the split, transformers fitted on test/validation data |
48
+
49
+ Every audit returns a 0-100 trust score. The CLI exits with code 1 on a high-severity leak, so it gates CI.
50
+
51
+ ## Demo
52
+
53
+ $ leaklens demo.csv --target is_fraud --arena
54
+ LeakLens trust score: 50/100
55
+ [HIGH] target_leakage :: chargeback_flag :: alone predicts target (roc_auc=1.000)
56
+ [MEDIUM] id_like :: txn_id :: 100.0% unique values; likely an identifier
57
+
58
+ Arena ranked by pr_auc (auto-dropped: chargeback_flag, txn_id)
59
+ model roc_auc pr_auc best_f1
60
+ neural_net 0.8374 0.4219 0.4353
61
+ linear 0.8410 0.4215 0.4304
62
+ tree_ensemble 0.8242 0.3562 0.4103
63
+
64
+ A leaked column makes accuracy look perfect. LeakLens drops it and shows the honest numbers.
65
+
66
+ ## Quick start
67
+
68
+ pip install -e “.[dev,app]”
69
+ leaklens train.csv --target label --test test.csv --time date
70
+ leaklens --scan notebooks/
71
+ leaklens train.csv --target label --test test.csv --group patient_id
72
+ python examples/make_demo.py
73
+ leaklens demo.csv --target is_fraud --arena
74
+ mlflow ui --backend-store-uri sqlite:///mlflow.db
75
+
76
+
77
+ ## Model arena
78
+ Auto-drops leaking and ID columns, then trains linear, gradient-boosted tree and neural-net models inside leak-safe pipelines (imputation, scaling and encoding are fitted on training data only). Runs are tracked in local MLflow (SQLite) and the winner is saved to `models/best.joblib`. Add `--time COL` for a temporal split.
79
+
80
+ ## API and UI
81
+
82
+ uvicorn leaklens.api:app --port 8000 # docs at /docs
83
+ streamlit run leaklens/ui.py
84
+
85
+ Endpoints: `GET /health`, `POST /audit`, `POST /scan`, `POST /predict`.
86
+
87
+ ## Docker
88
+
89
+ docker compose up --build
90
+
91
+ API on http://localhost:8000/docs, UI on http://localhost:8501.
92
+
93
+ ## GitHub Action
94
+ ```yaml
95
+ - uses: actions/checkout@v4
96
+ - uses: eyadstein/leaklens@main
97
+ with:
98
+ path: notebooks
99
+ ```
100
+
101
+ ## Pre-commit hook
102
+ ```yaml
103
+ repos:
104
+ - repo: https://github.com/eyadstein/leaklens
105
+ rev: main
106
+ hooks:
107
+ - id: leaklens
108
+ ```
109
+ Blocks a commit when a notebook or script fits a transformer before the split.
110
+
111
+ ## Trust score badge
112
+
113
+ leaklens --scan . --badge badge.json
114
+
115
+ Commit `badge.json`, then point a shields.io endpoint badge at its raw URL (as at the top of this file).
116
+
117
+ ## Architecture
118
+ ```mermaid
119
+ flowchart LR
120
+ A[CSV / notebooks] --> B[Data audit + leakage detectors]
121
+ A --> C[Static code scanner]
122
+ B --> D[Trust score]
123
+ C --> D
124
+ B --> E[Model arena + MLflow]
125
+ E --> F[best.joblib]
126
+ F --> G[FastAPI /predict]
127
+ D --> H[CLI / Streamlit / GitHub Action]
128
+ ```
129
+
130
+ ## Roadmap
131
+ - [x] Core detectors, static scanner, model arena, API, UI
132
+ - [x] Docker, GitHub Action, badge, CI
133
+ - [x] Group/entity leakage detector
134
+ - [x] PyPI packaging + release workflow (PyPI name `leaklens-ml`, command stays `leaklens`; create a GitHub release to publish)
135
+
136
+ MIT licensed.
@@ -0,0 +1,102 @@
1
+ # LeakLens
2
+
3
+ [![CI](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml/badge.svg)](https://github.com/eyadstein/leaklens/actions/workflows/ci.yml)
4
+ [![Leakage trust](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/eyadstein/leaklens/main/badge.json)](#trust-score-badge)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
6
+
7
+ **Catch data leakage before it fools you.** Most "99% accuracy" models are leaking. LeakLens audits both your **data** and your **code**, then proves what survives by training three model families on a leak-free pipeline.
8
+
9
+ ## What it catches
10
+ | Where | Leak |
11
+ |---|---|
12
+ | Data | Target leakage (a single feature that almost perfectly predicts the label), ID-like columns, train/test row overlap, temporal leakage, entity/group overlap (`--group COL`) |
13
+ | Code (`.py`, `.ipynb`) | Scaler/encoder/imputer fitted before the split, SMOTE before the split, transformers fitted on test/validation data |
14
+
15
+ Every audit returns a 0-100 trust score. The CLI exits with code 1 on a high-severity leak, so it gates CI.
16
+
17
+ ## Demo
18
+
19
+ $ leaklens demo.csv --target is_fraud --arena
20
+ LeakLens trust score: 50/100
21
+ [HIGH] target_leakage :: chargeback_flag :: alone predicts target (roc_auc=1.000)
22
+ [MEDIUM] id_like :: txn_id :: 100.0% unique values; likely an identifier
23
+
24
+ Arena ranked by pr_auc (auto-dropped: chargeback_flag, txn_id)
25
+ model roc_auc pr_auc best_f1
26
+ neural_net 0.8374 0.4219 0.4353
27
+ linear 0.8410 0.4215 0.4304
28
+ tree_ensemble 0.8242 0.3562 0.4103
29
+
30
+ A leaked column makes accuracy look perfect. LeakLens drops it and shows the honest numbers.
31
+
32
+ ## Quick start
33
+
34
+ pip install -e “.[dev,app]”
35
+ leaklens train.csv --target label --test test.csv --time date
36
+ leaklens --scan notebooks/
37
+ leaklens train.csv --target label --test test.csv --group patient_id
38
+ python examples/make_demo.py
39
+ leaklens demo.csv --target is_fraud --arena
40
+ mlflow ui --backend-store-uri sqlite:///mlflow.db
41
+
42
+
43
+ ## Model arena
44
+ Auto-drops leaking and ID columns, then trains linear, gradient-boosted tree and neural-net models inside leak-safe pipelines (imputation, scaling and encoding are fitted on training data only). Runs are tracked in local MLflow (SQLite) and the winner is saved to `models/best.joblib`. Add `--time COL` for a temporal split.
45
+
46
+ ## API and UI
47
+
48
+ uvicorn leaklens.api:app --port 8000 # docs at /docs
49
+ streamlit run leaklens/ui.py
50
+
51
+ Endpoints: `GET /health`, `POST /audit`, `POST /scan`, `POST /predict`.
52
+
53
+ ## Docker
54
+
55
+ docker compose up --build
56
+
57
+ API on http://localhost:8000/docs, UI on http://localhost:8501.
58
+
59
+ ## GitHub Action
60
+ ```yaml
61
+ - uses: actions/checkout@v4
62
+ - uses: eyadstein/leaklens@main
63
+ with:
64
+ path: notebooks
65
+ ```
66
+
67
+ ## Pre-commit hook
68
+ ```yaml
69
+ repos:
70
+ - repo: https://github.com/eyadstein/leaklens
71
+ rev: main
72
+ hooks:
73
+ - id: leaklens
74
+ ```
75
+ Blocks a commit when a notebook or script fits a transformer before the split.
76
+
77
+ ## Trust score badge
78
+
79
+ leaklens --scan . --badge badge.json
80
+
81
+ Commit `badge.json`, then point a shields.io endpoint badge at its raw URL (as at the top of this file).
82
+
83
+ ## Architecture
84
+ ```mermaid
85
+ flowchart LR
86
+ A[CSV / notebooks] --> B[Data audit + leakage detectors]
87
+ A --> C[Static code scanner]
88
+ B --> D[Trust score]
89
+ C --> D
90
+ B --> E[Model arena + MLflow]
91
+ E --> F[best.joblib]
92
+ F --> G[FastAPI /predict]
93
+ D --> H[CLI / Streamlit / GitHub Action]
94
+ ```
95
+
96
+ ## Roadmap
97
+ - [x] Core detectors, static scanner, model arena, API, UI
98
+ - [x] Docker, GitHub Action, badge, CI
99
+ - [x] Group/entity leakage detector
100
+ - [x] PyPI packaging + release workflow (PyPI name `leaklens-ml`, command stays `leaklens`; create a GitHub release to publish)
101
+
102
+ MIT licensed.
@@ -0,0 +1 @@
1
+ __version__ = "0.4.0"
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,77 @@
1
+ from __future__ import annotations
2
+
3
+ import io
4
+ from pathlib import Path
5
+
6
+ import joblib
7
+ import numpy as np
8
+ import pandas as pd
9
+ from fastapi import FastAPI, File, Form, HTTPException, UploadFile
10
+ from pydantic import BaseModel
11
+
12
+ from . import __version__
13
+ from .audit import audit
14
+ from .leakage import scan, score
15
+ from .scanner import scan_source
16
+
17
+ app = FastAPI(title="LeakLens", version=__version__)
18
+
19
+
20
+ class ScanReq(BaseModel):
21
+ source: str
22
+ name: str = "snippet.py"
23
+
24
+
25
+ class PredictReq(BaseModel):
26
+ rows: list[dict]
27
+ model_path: str = "models/best.joblib"
28
+
29
+
30
+ @app.get("/health")
31
+ def health():
32
+ return {"status": "ok", "version": __version__}
33
+
34
+
35
+ @app.post("/audit")
36
+ async def audit_csv(file: UploadFile = File(...), target: str = Form(...), threshold: float = Form(0.95)):
37
+ try:
38
+ df = pd.read_csv(io.BytesIO(await file.read()))
39
+ except Exception as e:
40
+ raise HTTPException(400, f"bad CSV: {e}")
41
+ if target not in df.columns:
42
+ raise HTTPException(422, f"target '{target}' not in columns")
43
+ fs = scan(df, target, threshold)
44
+ return {"trust_score": score(fs), "audit": audit(df, target), "findings": [f.to_dict() for f in fs]}
45
+
46
+
47
+ @app.post("/scan")
48
+ def scan_code(req: ScanReq):
49
+ fs = scan_source(req.source, req.name)
50
+ return {"trust_score": score(fs), "findings": [f.to_dict() for f in fs]}
51
+
52
+
53
+ @app.post("/predict")
54
+ def predict(req: PredictReq):
55
+ p = Path(req.model_path)
56
+ if not p.exists():
57
+ raise HTTPException(404, "model not found; run: leaklens data.csv --target T --arena")
58
+ b = joblib.load(p)
59
+ df = pd.DataFrame(req.rows)
60
+ miss = [c for c in b["features"] if c not in df.columns]
61
+ if miss:
62
+ raise HTTPException(422, f"missing features: {miss}")
63
+ X = df[b["features"]].copy()
64
+ pipe = b["pipeline"]
65
+ tf = pipe.named_steps["prep"].transformers_
66
+ for c in tf[0][2]:
67
+ X[c] = pd.to_numeric(X[c], errors="coerce")
68
+ for c in tf[1][2]:
69
+ X[c] = X[c].astype(str)
70
+ if b["task"] == "classification":
71
+ out = []
72
+ for pr in pipe.predict_proba(X):
73
+ i = int(np.argmax(pr))
74
+ out.append({"label": b["classes"][i], "score": float(pr[1] if len(pr) == 2 else pr[i])})
75
+ else:
76
+ out = [{"value": float(v)} for v in pipe.predict(X)]
77
+ return {"model": b["model"], "task": b["task"], "predictions": out}
@@ -0,0 +1,140 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ import joblib
6
+ import mlflow
7
+ import numpy as np
8
+ import pandas as pd
9
+ from sklearn.compose import ColumnTransformer
10
+ from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
11
+ from sklearn.impute import SimpleImputer
12
+ from sklearn.linear_model import LogisticRegression, Ridge
13
+ from sklearn.metrics import (accuracy_score, average_precision_score, f1_score, mean_absolute_error,
14
+ mean_squared_error, precision_recall_curve, precision_score, r2_score,
15
+ recall_score, roc_auc_score)
16
+ from sklearn.model_selection import train_test_split
17
+ from sklearn.neural_network import MLPClassifier, MLPRegressor
18
+ from sklearn.pipeline import Pipeline
19
+ from sklearn.preprocessing import OneHotEncoder, StandardScaler
20
+
21
+ from .leakage import scan
22
+
23
+
24
+ def _models(clf, seed):
25
+ if clf:
26
+ return {
27
+ "linear": LogisticRegression(max_iter=1000, class_weight="balanced"),
28
+ "tree_ensemble": HistGradientBoostingClassifier(random_state=seed),
29
+ "neural_net": MLPClassifier(hidden_layer_sizes=(64, 32), max_iter=300, early_stopping=True, random_state=seed),
30
+ }
31
+ return {
32
+ "linear": Ridge(),
33
+ "tree_ensemble": HistGradientBoostingRegressor(random_state=seed),
34
+ "neural_net": MLPRegressor(hidden_layer_sizes=(64, 32), max_iter=300, early_stopping=True, random_state=seed),
35
+ }
36
+
37
+
38
+ def _prep(num, cat):
39
+ return ColumnTransformer([
40
+ ("num", Pipeline([("imp", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), num),
41
+ ("cat", OneHotEncoder(handle_unknown="infrequent_if_exist", max_categories=20, sparse_output=False), cat),
42
+ ])
43
+
44
+
45
+ def _metrics(clf, y, pipe, X):
46
+ out = {}
47
+ pred = pipe.predict(X)
48
+ if clf:
49
+ proba = pipe.predict_proba(X)
50
+ if proba.shape[1] == 2:
51
+ s = proba[:, 1]
52
+ try:
53
+ out["roc_auc"] = roc_auc_score(y, s)
54
+ out["pr_auc"] = average_precision_score(y, s)
55
+ p, r, _ = precision_recall_curve(y, s)
56
+ out["best_f1"] = float((2 * p * r / np.maximum(p + r, 1e-12)).max())
57
+ except ValueError:
58
+ pass
59
+ out["f1"] = f1_score(y, pred, zero_division=0)
60
+ out["precision"] = precision_score(y, pred, zero_division=0)
61
+ out["recall"] = recall_score(y, pred, zero_division=0)
62
+ else:
63
+ out["f1_macro"] = f1_score(y, pred, average="macro", zero_division=0)
64
+ out["accuracy"] = accuracy_score(y, pred)
65
+ else:
66
+ out["mae"] = mean_absolute_error(y, pred)
67
+ out["rmse"] = float(np.sqrt(mean_squared_error(y, pred)))
68
+ out["r2"] = r2_score(y, pred)
69
+ return {k: float(v) for k, v in out.items()}
70
+
71
+
72
+ def run(df, target, time_col=None, out_dir="models", tracking_uri="sqlite:///mlflow.db",
73
+ experiment="leaklens-arena", drop_leaks=True, seed=0):
74
+ d = df[df[target].notna()].copy()
75
+ if len(d) < 50:
76
+ raise ValueError("need at least 50 labeled rows")
77
+ dropped = []
78
+ if drop_leaks:
79
+ dropped = [f.column for f in scan(d, target)
80
+ if f.kind == "id_like" or (f.kind == "target_leakage" and f.severity == "high")]
81
+ y_raw = d[target]
82
+ clf = y_raw.dtype.kind not in "iuf" or y_raw.nunique() <= 20
83
+ if clf:
84
+ cats = y_raw.astype("category")
85
+ y = cats.cat.codes.to_numpy()
86
+ classes = [str(c) for c in cats.cat.categories]
87
+ else:
88
+ y = y_raw.astype(float).to_numpy()
89
+ classes = []
90
+ drop = {target, *dropped}
91
+ if time_col:
92
+ drop.add(time_col)
93
+ X = d.drop(columns=[c for c in drop if c in d.columns])
94
+ X = X.drop(columns=X.select_dtypes(include=["datetime", "datetimetz"]).columns)
95
+ for c in X.select_dtypes("bool").columns:
96
+ X[c] = X[c].astype(int)
97
+ num = X.select_dtypes(include=["number"]).columns.tolist()
98
+ cat = [c for c in X.columns if c not in num]
99
+ if num:
100
+ X[num] = X[num].replace([np.inf, -np.inf], np.nan)
101
+ for c in cat:
102
+ X[c] = X[c].astype(str)
103
+ n = len(d)
104
+ if time_col:
105
+ t = pd.to_datetime(d[time_col], errors="coerce").to_numpy()
106
+ order = np.argsort(t, kind="stable")
107
+ cut = int(n * 0.8)
108
+ tr_i, te_i = order[:cut], order[cut:]
109
+ else:
110
+ strat = y if clf and np.bincount(y).min() >= 2 else None
111
+ tr_i, te_i = train_test_split(np.arange(n), test_size=0.2, stratify=strat, random_state=seed)
112
+ if clf and len(np.unique(y[tr_i])) < 2:
113
+ raise ValueError("training split has a single class")
114
+ Xtr, Xte, ytr, yte = X.iloc[tr_i], X.iloc[te_i], y[tr_i], y[te_i]
115
+ primary = ("pr_auc" if len(classes) == 2 else "f1_macro") if clf else "rmse"
116
+ mlflow.set_tracking_uri(tracking_uri)
117
+ mlflow.set_experiment(experiment)
118
+ rows, pipes = [], {}
119
+ for name, model in _models(clf, seed).items():
120
+ pipe = Pipeline([("prep", _prep(num, cat)), ("model", model)])
121
+ with mlflow.start_run(run_name=name):
122
+ mlflow.log_params({"model": name, "task": "classification" if clf else "regression",
123
+ "split": "temporal" if time_col else "random", "n_train": len(tr_i),
124
+ "n_test": len(te_i), "dropped": ",".join(map(str, dropped)) or "none"})
125
+ pipe.fit(Xtr, ytr)
126
+ m = _metrics(clf, yte, pipe, Xte)
127
+ mlflow.log_metrics({k: v for k, v in m.items() if np.isfinite(v)})
128
+ pipes[name] = pipe
129
+ rows.append({"model": name, **m})
130
+ res = pd.DataFrame(rows)
131
+ if primary not in res:
132
+ res[primary] = np.nan
133
+ res = res.sort_values(primary, ascending=(primary == "rmse"), na_position="last").reset_index(drop=True)
134
+ Path(out_dir).mkdir(parents=True, exist_ok=True)
135
+ joblib.dump({"pipeline": pipes[res.loc[0, "model"]], "features": list(X.columns), "target": target,
136
+ "task": "classification" if clf else "regression", "classes": classes,
137
+ "dropped": dropped, "model": res.loc[0, "model"]}, Path(out_dir) / "best.joblib")
138
+ res.attrs["dropped"] = dropped
139
+ res.attrs["primary"] = primary
140
+ return res
@@ -0,0 +1,18 @@
1
+ from __future__ import annotations
2
+
3
+ import pandas as pd
4
+
5
+
6
+ def audit(df: pd.DataFrame, target: str | None = None) -> dict:
7
+ out = {
8
+ "rows": int(len(df)),
9
+ "cols": int(df.shape[1]),
10
+ "duplicate_rows": int(df.duplicated().sum()),
11
+ "null_rate": {c: round(float(v), 4) for c, v in df.isna().mean().items() if v > 0},
12
+ "cardinality": {c: int(df[c].nunique(dropna=True)) for c in df.columns},
13
+ }
14
+ if target is not None and target in df.columns:
15
+ vc = df[target].value_counts(normalize=True)
16
+ out["target_distribution"] = {str(k): round(float(v), 4) for k, v in vc.items()}
17
+ out["minority_rate"] = round(float(vc.min()), 6)
18
+ return out
@@ -0,0 +1,75 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+
6
+ import pandas as pd
7
+
8
+ from .audit import audit
9
+ from .leakage import group_overlap, group_risk, scan, score, split_overlap, temporal_overlap
10
+ from .scanner import scan_path
11
+
12
+
13
+ def main(argv=None) -> int:
14
+ p = argparse.ArgumentParser(prog="leaklens", description="Audit datasets and ML code for leakage")
15
+ p.add_argument("train", nargs="?")
16
+ p.add_argument("--target")
17
+ p.add_argument("--test")
18
+ p.add_argument("--time")
19
+ p.add_argument("--group", help="entity column (patient, customer...) to check for group leakage")
20
+ p.add_argument("--threshold", type=float, default=0.95)
21
+ p.add_argument("--scan", nargs="+", help="file or folder of .py/.ipynb to scan for pipeline leaks")
22
+ p.add_argument("--arena", action="store_true", help="compare 3 model families, tracked in MLflow")
23
+ p.add_argument("--out", default="models")
24
+ p.add_argument("--tracking-uri", default="sqlite:///mlflow.db")
25
+ p.add_argument("--json", action="store_true")
26
+ p.add_argument("--badge", help="write a shields.io endpoint JSON with the trust score")
27
+ a = p.parse_args(argv)
28
+ if not a.train and not a.scan:
29
+ p.error("give a CSV and/or --scan PATH")
30
+ if a.train and not a.target:
31
+ p.error("--target is required with a CSV")
32
+ if a.arena and not a.train:
33
+ p.error("--arena needs a CSV")
34
+ fs, rep, res = [], {}, None
35
+ if a.train:
36
+ tr = pd.read_csv(a.train)
37
+ fs += scan(tr, a.target, a.threshold)
38
+ if a.test:
39
+ te = pd.read_csv(a.test)
40
+ fs += split_overlap(tr, te)
41
+ if a.time:
42
+ fs += temporal_overlap(tr, te, a.time)
43
+ if a.group:
44
+ fs += group_risk(tr, a.group)
45
+ if a.test:
46
+ fs += group_overlap(tr, te, a.group)
47
+ rep["audit"] = audit(tr, a.target)
48
+ if a.arena:
49
+ try:
50
+ from .arena import run
51
+ except ImportError:
52
+ p.error('install extras: pip install -e ".[arena]"')
53
+ res = run(tr, a.target, a.time, a.out, a.tracking_uri)
54
+ rep["arena"] = res.round(4).to_dict("records")
55
+ if a.scan:
56
+ for s in a.scan:
57
+ fs += scan_path(s)
58
+ rep = {"trust_score": score(fs), **rep, "findings": [f.to_dict() for f in fs]}
59
+ if a.badge:
60
+ s = rep["trust_score"]
61
+ col = "brightgreen" if s >= 85 else "yellow" if s >= 60 else "red"
62
+ with open(a.badge, "w", encoding="utf-8") as fh:
63
+ json.dump({"schemaVersion": 1, "label": "leakage trust", "message": f"{s}/100", "color": col}, fh)
64
+ if a.json:
65
+ print(json.dumps(rep, indent=2))
66
+ else:
67
+ print(f"LeakLens trust score: {rep['trust_score']}/100")
68
+ for f in fs:
69
+ print(f"[{f.severity.upper()}] {f.kind} :: {f.column} :: {f.detail}")
70
+ if not fs:
71
+ print("No leakage found.")
72
+ if res is not None:
73
+ print(f"\nArena ranked by {res.attrs['primary']} (auto-dropped: {', '.join(map(str, res.attrs['dropped'])) or 'none'})")
74
+ print(res.round(4).to_string(index=False))
75
+ return 1 if any(f.severity == "high" for f in fs) else 0