ml-engineer-skills 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_engineer_skills-0.4.0/LICENSE +21 -0
- ml_engineer_skills-0.4.0/PKG-INFO +167 -0
- ml_engineer_skills-0.4.0/README.md +145 -0
- ml_engineer_skills-0.4.0/pyproject.toml +40 -0
- ml_engineer_skills-0.4.0/setup.cfg +4 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/PKG-INFO +167 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/SOURCES.txt +38 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/dependency_links.txt +1 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/entry_points.txt +5 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/requires.txt +9 -0
- ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/top_level.txt +1 -0
- ml_engineer_skills-0.4.0/src/mlcheck/__init__.py +52 -0
- ml_engineer_skills-0.4.0/src/mlcheck/__main__.py +5 -0
- ml_engineer_skills-0.4.0/src/mlcheck/baselines.py +124 -0
- ml_engineer_skills-0.4.0/src/mlcheck/cli.py +293 -0
- ml_engineer_skills-0.4.0/src/mlcheck/core.py +153 -0
- ml_engineer_skills-0.4.0/src/mlcheck/detectors.py +192 -0
- ml_engineer_skills-0.4.0/src/mlcheck/fixes.py +57 -0
- ml_engineer_skills-0.4.0/src/mlcheck/impact.py +73 -0
- ml_engineer_skills-0.4.0/src/mlcheck/integrations/__init__.py +7 -0
- ml_engineer_skills-0.4.0/src/mlcheck/integrations/jupyter.py +41 -0
- ml_engineer_skills-0.4.0/src/mlcheck/integrations/pytest_plugin.py +79 -0
- ml_engineer_skills-0.4.0/src/mlcheck/leakage.py +201 -0
- ml_engineer_skills-0.4.0/src/mlcheck/metrics.py +82 -0
- ml_engineer_skills-0.4.0/src/mlcheck/repro.py +120 -0
- ml_engineer_skills-0.4.0/src/mlcheck/splits.py +120 -0
- ml_engineer_skills-0.4.0/tests/test_baselines_metrics_extra.py +37 -0
- ml_engineer_skills-0.4.0/tests/test_choosing_evaluation_metrics.py +26 -0
- ml_engineer_skills-0.4.0/tests/test_cli.py +54 -0
- ml_engineer_skills-0.4.0/tests/test_cli_v03.py +48 -0
- ml_engineer_skills-0.4.0/tests/test_core.py +28 -0
- ml_engineer_skills-0.4.0/tests/test_designing_validation_splits.py +33 -0
- ml_engineer_skills-0.4.0/tests/test_detecting_data_leakage.py +42 -0
- ml_engineer_skills-0.4.0/tests/test_detectors.py +53 -0
- ml_engineer_skills-0.4.0/tests/test_establishing_baselines.py +32 -0
- ml_engineer_skills-0.4.0/tests/test_impact.py +23 -0
- ml_engineer_skills-0.4.0/tests/test_integrations.py +51 -0
- ml_engineer_skills-0.4.0/tests/test_leakage_patterns.py +46 -0
- ml_engineer_skills-0.4.0/tests/test_output_and_fixes.py +61 -0
- ml_engineer_skills-0.4.0/tests/test_repro.py +29 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Krishi Shah
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ml-engineer-skills
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command.
|
|
5
|
+
Author: Krishi Shah
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/krishi-shah/ml-engineer-skills
|
|
8
|
+
Project-URL: Repository, https://github.com/krishi-shah/ml-engineer-skills
|
|
9
|
+
Project-URL: Issues, https://github.com/krishi-shah/ml-engineer-skills/issues
|
|
10
|
+
Keywords: machine-learning,ml-engineering,data-leakage,model-validation,mlops,scikit-learn
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy>=1.24
|
|
15
|
+
Requires-Dist: pandas>=2.0
|
|
16
|
+
Requires-Dist: scikit-learn>=1.3
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
19
|
+
Provides-Extra: notebook
|
|
20
|
+
Requires-Dist: ipython; extra == "notebook"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# ML Engineer Skills
|
|
24
|
+
|
|
25
|
+
[](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
|
|
26
|
+
[](https://pypi.org/project/ml-engineer-skills/)
|
|
27
|
+
[](https://www.python.org/)
|
|
28
|
+
[](LICENSE)
|
|
29
|
+
|
|
30
|
+
**A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
|
|
31
|
+
|
|
32
|
+
<!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
|
|
33
|
+
<!--  -->
|
|
34
|
+
|
|
35
|
+
## Why you'd want this
|
|
36
|
+
|
|
37
|
+
When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
|
|
38
|
+
|
|
39
|
+
`mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install ml-engineer-skills
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
|
|
48
|
+
|
|
49
|
+
## Try it in 30 seconds
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python example/quickstart.py
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
You'll see it catch 5 real bugs at once, each with a fix:
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
[x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
|
|
59
|
+
-> fix: Move the transform into a Pipeline fit AFTER the split.
|
|
60
|
+
[x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
|
|
61
|
+
[x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
|
|
62
|
+
...
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Use it on your own project
|
|
66
|
+
|
|
67
|
+
Check a training script for leaks:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
mlcheck scan-source train.py
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
mlcheck audit --source train.py --train train.csv --test test.csv \
|
|
77
|
+
--target label --fail-on error
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Want the corrected code, not just the warning? Add `--suggest`:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
mlcheck scan-source train.py --suggest
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## What it checks for
|
|
87
|
+
|
|
88
|
+
| Mistake | What it means (plain English) |
|
|
89
|
+
|---|---|
|
|
90
|
+
| **Data leakage** | Your model secretly saw information it won't have in real life. |
|
|
91
|
+
| **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
|
|
92
|
+
| **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
|
|
93
|
+
| **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
|
|
94
|
+
| **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
|
|
95
|
+
| **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
|
|
96
|
+
|
|
97
|
+
## Know your *real* score
|
|
98
|
+
|
|
99
|
+
This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from mlcheck import split_impact
|
|
103
|
+
for finding in split_impact(model, X, y, group_col="user_id"):
|
|
104
|
+
print(finding)
|
|
105
|
+
# Naive split scores 1.000, but an honest split scores 0.733.
|
|
106
|
+
# Your real number is about 0.733.
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
That honest number is the one that holds up in the real world.
|
|
110
|
+
|
|
111
|
+
## Run it automatically (so you never forget)
|
|
112
|
+
|
|
113
|
+
**In your tests** — fail the build if there's a leak:
|
|
114
|
+
```python
|
|
115
|
+
from mlcheck.integrations.pytest_plugin import assert_no_leakage
|
|
116
|
+
def test_no_leakage():
|
|
117
|
+
assert_no_leakage("train.py")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Before every commit** — add to `.pre-commit-config.yaml`:
|
|
121
|
+
```yaml
|
|
122
|
+
- repo: https://github.com/krishi-shah/ml-engineer-skills
|
|
123
|
+
rev: v0.4.0
|
|
124
|
+
hooks:
|
|
125
|
+
- id: mlcheck-scan-source
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**In a Jupyter notebook** — flag leaks as you write them:
|
|
129
|
+
```python
|
|
130
|
+
%load_ext mlcheck.integrations.jupyter
|
|
131
|
+
%%mlcheck
|
|
132
|
+
scaler = StandardScaler().fit(X) # flagged right after the cell runs
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
|
|
136
|
+
```
|
|
137
|
+
/plugin marketplace add krishi-shah/ml-engineer-skills
|
|
138
|
+
/plugin install ml-engineer-skills
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Is it for me?
|
|
142
|
+
|
|
143
|
+
- **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
|
|
144
|
+
- **On a team?** Yes — wire it into CI so a leaked model can't get merged.
|
|
145
|
+
- **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
|
|
146
|
+
|
|
147
|
+
It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
|
|
148
|
+
|
|
149
|
+
## What's inside
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
|
|
153
|
+
skills/ the playbooks that teach each check (and guide an AI assistant)
|
|
154
|
+
example/ quickstart.py + a deliberately-buggy script to scan
|
|
155
|
+
tests/ 64 tests, each proving a check catches a real bug
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Verify it yourself:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
pip install -e ".[dev]"
|
|
162
|
+
pytest -q # 64 passed
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## License
|
|
166
|
+
|
|
167
|
+
MIT
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# ML Engineer Skills
|
|
2
|
+
|
|
3
|
+
[](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/ml-engineer-skills/)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
|
|
9
|
+
|
|
10
|
+
<!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
|
|
11
|
+
<!--  -->
|
|
12
|
+
|
|
13
|
+
## Why you'd want this
|
|
14
|
+
|
|
15
|
+
When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
|
|
16
|
+
|
|
17
|
+
`mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install ml-engineer-skills
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
|
|
26
|
+
|
|
27
|
+
## Try it in 30 seconds
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
python example/quickstart.py
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
You'll see it catch 5 real bugs at once, each with a fix:
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
[x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
|
|
37
|
+
-> fix: Move the transform into a Pipeline fit AFTER the split.
|
|
38
|
+
[x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
|
|
39
|
+
[x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
|
|
40
|
+
...
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Use it on your own project
|
|
44
|
+
|
|
45
|
+
Check a training script for leaks:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
mlcheck scan-source train.py
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
mlcheck audit --source train.py --train train.csv --test test.csv \
|
|
55
|
+
--target label --fail-on error
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Want the corrected code, not just the warning? Add `--suggest`:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
mlcheck scan-source train.py --suggest
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## What it checks for
|
|
65
|
+
|
|
66
|
+
| Mistake | What it means (plain English) |
|
|
67
|
+
|---|---|
|
|
68
|
+
| **Data leakage** | Your model secretly saw information it won't have in real life. |
|
|
69
|
+
| **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
|
|
70
|
+
| **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
|
|
71
|
+
| **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
|
|
72
|
+
| **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
|
|
73
|
+
| **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
|
|
74
|
+
|
|
75
|
+
## Know your *real* score
|
|
76
|
+
|
|
77
|
+
This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from mlcheck import split_impact
|
|
81
|
+
for finding in split_impact(model, X, y, group_col="user_id"):
|
|
82
|
+
print(finding)
|
|
83
|
+
# Naive split scores 1.000, but an honest split scores 0.733.
|
|
84
|
+
# Your real number is about 0.733.
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
That honest number is the one that holds up in the real world.
|
|
88
|
+
|
|
89
|
+
## Run it automatically (so you never forget)
|
|
90
|
+
|
|
91
|
+
**In your tests** — fail the build if there's a leak:
|
|
92
|
+
```python
|
|
93
|
+
from mlcheck.integrations.pytest_plugin import assert_no_leakage
|
|
94
|
+
def test_no_leakage():
|
|
95
|
+
assert_no_leakage("train.py")
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**Before every commit** — add to `.pre-commit-config.yaml`:
|
|
99
|
+
```yaml
|
|
100
|
+
- repo: https://github.com/krishi-shah/ml-engineer-skills
|
|
101
|
+
rev: v0.4.0
|
|
102
|
+
hooks:
|
|
103
|
+
- id: mlcheck-scan-source
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
**In a Jupyter notebook** — flag leaks as you write them:
|
|
107
|
+
```python
|
|
108
|
+
%load_ext mlcheck.integrations.jupyter
|
|
109
|
+
%%mlcheck
|
|
110
|
+
scaler = StandardScaler().fit(X) # flagged right after the cell runs
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
|
|
114
|
+
```
|
|
115
|
+
/plugin marketplace add krishi-shah/ml-engineer-skills
|
|
116
|
+
/plugin install ml-engineer-skills
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Is it for me?
|
|
120
|
+
|
|
121
|
+
- **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
|
|
122
|
+
- **On a team?** Yes — wire it into CI so a leaked model can't get merged.
|
|
123
|
+
- **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
|
|
124
|
+
|
|
125
|
+
It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
|
|
126
|
+
|
|
127
|
+
## What's inside
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
|
|
131
|
+
skills/ the playbooks that teach each check (and guide an AI assistant)
|
|
132
|
+
example/ quickstart.py + a deliberately-buggy script to scan
|
|
133
|
+
tests/ 64 tests, each proving a check catches a real bug
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Verify it yourself:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
pytest -q # 64 passed
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
## License
|
|
144
|
+
|
|
145
|
+
MIT
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ml-engineer-skills"
|
|
7
|
+
version = "0.4.0"
|
|
8
|
+
description = "A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Krishi Shah" }]
|
|
13
|
+
keywords = ["machine-learning", "ml-engineering", "data-leakage", "model-validation", "mlops", "scikit-learn"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"numpy>=1.24",
|
|
16
|
+
"pandas>=2.0",
|
|
17
|
+
"scikit-learn>=1.3",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/krishi-shah/ml-engineer-skills"
|
|
22
|
+
Repository = "https://github.com/krishi-shah/ml-engineer-skills"
|
|
23
|
+
Issues = "https://github.com/krishi-shah/ml-engineer-skills/issues"
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = ["pytest>=7.4"]
|
|
27
|
+
notebook = ["ipython"]
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
mlcheck = "mlcheck.cli:main"
|
|
31
|
+
|
|
32
|
+
[project.entry-points.pytest11]
|
|
33
|
+
mlcheck = "mlcheck.integrations.pytest_plugin"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
pythonpath = ["src"]
|
|
40
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ml-engineer-skills
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command.
|
|
5
|
+
Author: Krishi Shah
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/krishi-shah/ml-engineer-skills
|
|
8
|
+
Project-URL: Repository, https://github.com/krishi-shah/ml-engineer-skills
|
|
9
|
+
Project-URL: Issues, https://github.com/krishi-shah/ml-engineer-skills/issues
|
|
10
|
+
Keywords: machine-learning,ml-engineering,data-leakage,model-validation,mlops,scikit-learn
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy>=1.24
|
|
15
|
+
Requires-Dist: pandas>=2.0
|
|
16
|
+
Requires-Dist: scikit-learn>=1.3
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
19
|
+
Provides-Extra: notebook
|
|
20
|
+
Requires-Dist: ipython; extra == "notebook"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# ML Engineer Skills
|
|
24
|
+
|
|
25
|
+
[](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
|
|
26
|
+
[](https://pypi.org/project/ml-engineer-skills/)
|
|
27
|
+
[](https://www.python.org/)
|
|
28
|
+
[](LICENSE)
|
|
29
|
+
|
|
30
|
+
**A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
|
|
31
|
+
|
|
32
|
+
<!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
|
|
33
|
+
<!--  -->
|
|
34
|
+
|
|
35
|
+
## Why you'd want this
|
|
36
|
+
|
|
37
|
+
When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
|
|
38
|
+
|
|
39
|
+
`mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install ml-engineer-skills
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
|
|
48
|
+
|
|
49
|
+
## Try it in 30 seconds
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python example/quickstart.py
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
You'll see it catch 5 real bugs at once, each with a fix:
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
[x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
|
|
59
|
+
-> fix: Move the transform into a Pipeline fit AFTER the split.
|
|
60
|
+
[x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
|
|
61
|
+
[x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
|
|
62
|
+
...
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Use it on your own project
|
|
66
|
+
|
|
67
|
+
Check a training script for leaks:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
mlcheck scan-source train.py
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
mlcheck audit --source train.py --train train.csv --test test.csv \
|
|
77
|
+
--target label --fail-on error
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Want the corrected code, not just the warning? Add `--suggest`:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
mlcheck scan-source train.py --suggest
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## What it checks for
|
|
87
|
+
|
|
88
|
+
| Mistake | What it means (plain English) |
|
|
89
|
+
|---|---|
|
|
90
|
+
| **Data leakage** | Your model secretly saw information it won't have in real life. |
|
|
91
|
+
| **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
|
|
92
|
+
| **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
|
|
93
|
+
| **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
|
|
94
|
+
| **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
|
|
95
|
+
| **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
|
|
96
|
+
|
|
97
|
+
## Know your *real* score
|
|
98
|
+
|
|
99
|
+
This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from mlcheck import split_impact
|
|
103
|
+
for finding in split_impact(model, X, y, group_col="user_id"):
|
|
104
|
+
print(finding)
|
|
105
|
+
# Naive split scores 1.000, but an honest split scores 0.733.
|
|
106
|
+
# Your real number is about 0.733.
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
That honest number is the one that holds up in the real world.
|
|
110
|
+
|
|
111
|
+
## Run it automatically (so you never forget)
|
|
112
|
+
|
|
113
|
+
**In your tests** — fail the build if there's a leak:
|
|
114
|
+
```python
|
|
115
|
+
from mlcheck.integrations.pytest_plugin import assert_no_leakage
|
|
116
|
+
def test_no_leakage():
|
|
117
|
+
assert_no_leakage("train.py")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Before every commit** — add to `.pre-commit-config.yaml`:
|
|
121
|
+
```yaml
|
|
122
|
+
- repo: https://github.com/krishi-shah/ml-engineer-skills
|
|
123
|
+
rev: v0.4.0
|
|
124
|
+
hooks:
|
|
125
|
+
- id: mlcheck-scan-source
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**In a Jupyter notebook** — flag leaks as you write them:
|
|
129
|
+
```python
|
|
130
|
+
%load_ext mlcheck.integrations.jupyter
|
|
131
|
+
%%mlcheck
|
|
132
|
+
scaler = StandardScaler().fit(X) # flagged right after the cell runs
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
|
|
136
|
+
```
|
|
137
|
+
/plugin marketplace add krishi-shah/ml-engineer-skills
|
|
138
|
+
/plugin install ml-engineer-skills
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Is it for me?
|
|
142
|
+
|
|
143
|
+
- **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
|
|
144
|
+
- **On a team?** Yes — wire it into CI so a leaked model can't get merged.
|
|
145
|
+
- **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
|
|
146
|
+
|
|
147
|
+
It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
|
|
148
|
+
|
|
149
|
+
## What's inside
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
|
|
153
|
+
skills/ the playbooks that teach each check (and guide an AI assistant)
|
|
154
|
+
example/ quickstart.py + a deliberately-buggy script to scan
|
|
155
|
+
tests/ 64 tests, each proving a check catches a real bug
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Verify it yourself:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
pip install -e ".[dev]"
|
|
162
|
+
pytest -q # 64 passed
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## License
|
|
166
|
+
|
|
167
|
+
MIT
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/ml_engineer_skills.egg-info/PKG-INFO
|
|
5
|
+
src/ml_engineer_skills.egg-info/SOURCES.txt
|
|
6
|
+
src/ml_engineer_skills.egg-info/dependency_links.txt
|
|
7
|
+
src/ml_engineer_skills.egg-info/entry_points.txt
|
|
8
|
+
src/ml_engineer_skills.egg-info/requires.txt
|
|
9
|
+
src/ml_engineer_skills.egg-info/top_level.txt
|
|
10
|
+
src/mlcheck/__init__.py
|
|
11
|
+
src/mlcheck/__main__.py
|
|
12
|
+
src/mlcheck/baselines.py
|
|
13
|
+
src/mlcheck/cli.py
|
|
14
|
+
src/mlcheck/core.py
|
|
15
|
+
src/mlcheck/detectors.py
|
|
16
|
+
src/mlcheck/fixes.py
|
|
17
|
+
src/mlcheck/impact.py
|
|
18
|
+
src/mlcheck/leakage.py
|
|
19
|
+
src/mlcheck/metrics.py
|
|
20
|
+
src/mlcheck/repro.py
|
|
21
|
+
src/mlcheck/splits.py
|
|
22
|
+
src/mlcheck/integrations/__init__.py
|
|
23
|
+
src/mlcheck/integrations/jupyter.py
|
|
24
|
+
src/mlcheck/integrations/pytest_plugin.py
|
|
25
|
+
tests/test_baselines_metrics_extra.py
|
|
26
|
+
tests/test_choosing_evaluation_metrics.py
|
|
27
|
+
tests/test_cli.py
|
|
28
|
+
tests/test_cli_v03.py
|
|
29
|
+
tests/test_core.py
|
|
30
|
+
tests/test_designing_validation_splits.py
|
|
31
|
+
tests/test_detecting_data_leakage.py
|
|
32
|
+
tests/test_detectors.py
|
|
33
|
+
tests/test_establishing_baselines.py
|
|
34
|
+
tests/test_impact.py
|
|
35
|
+
tests/test_integrations.py
|
|
36
|
+
tests/test_leakage_patterns.py
|
|
37
|
+
tests/test_output_and_fixes.py
|
|
38
|
+
tests/test_repro.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mlcheck
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""mlcheck — the ML rigor toolkit.
|
|
2
|
+
|
|
3
|
+
Runnable diagnostics that catch the silent mistakes between "works in the
|
|
4
|
+
notebook" and "works in production": unbeaten baselines, broken validation
|
|
5
|
+
splits, data leakage, and metric malpractice.
|
|
6
|
+
|
|
7
|
+
Public API:
|
|
8
|
+
from mlcheck import (
|
|
9
|
+
Finding, Severity, format_report, worst_severity,
|
|
10
|
+
evaluate_baseline, audit_split, scan_source, scan_data, advise_metric,
|
|
11
|
+
)
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from .core import Finding, Severity, format_report, format_json, format_markdown, worst_severity
|
|
16
|
+
from .baselines import evaluate_baseline, baseline_score
|
|
17
|
+
from .splits import audit_split
|
|
18
|
+
from .leakage import scan_source, scan_data
|
|
19
|
+
from .metrics import advise_metric
|
|
20
|
+
from .detectors import (
|
|
21
|
+
adversarial_validation,
|
|
22
|
+
signal_is_real,
|
|
23
|
+
target_leak_scan,
|
|
24
|
+
can_overfit_small_sample,
|
|
25
|
+
)
|
|
26
|
+
from .repro import scan_reproducibility, data_fingerprint
|
|
27
|
+
from .impact import split_impact
|
|
28
|
+
|
|
29
|
+
__version__ = "0.4.0"
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"Finding",
|
|
33
|
+
"Severity",
|
|
34
|
+
"format_report",
|
|
35
|
+
"format_json",
|
|
36
|
+
"format_markdown",
|
|
37
|
+
"worst_severity",
|
|
38
|
+
"split_impact",
|
|
39
|
+
"evaluate_baseline",
|
|
40
|
+
"baseline_score",
|
|
41
|
+
"audit_split",
|
|
42
|
+
"scan_source",
|
|
43
|
+
"scan_data",
|
|
44
|
+
"advise_metric",
|
|
45
|
+
"adversarial_validation",
|
|
46
|
+
"signal_is_real",
|
|
47
|
+
"target_leak_scan",
|
|
48
|
+
"can_overfit_small_sample",
|
|
49
|
+
"scan_reproducibility",
|
|
50
|
+
"data_fingerprint",
|
|
51
|
+
"__version__",
|
|
52
|
+
]
|