ml-engineer-skills 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. ml_engineer_skills-0.4.0/LICENSE +21 -0
  2. ml_engineer_skills-0.4.0/PKG-INFO +167 -0
  3. ml_engineer_skills-0.4.0/README.md +145 -0
  4. ml_engineer_skills-0.4.0/pyproject.toml +40 -0
  5. ml_engineer_skills-0.4.0/setup.cfg +4 -0
  6. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/PKG-INFO +167 -0
  7. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/SOURCES.txt +38 -0
  8. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/dependency_links.txt +1 -0
  9. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/entry_points.txt +5 -0
  10. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/requires.txt +9 -0
  11. ml_engineer_skills-0.4.0/src/ml_engineer_skills.egg-info/top_level.txt +1 -0
  12. ml_engineer_skills-0.4.0/src/mlcheck/__init__.py +52 -0
  13. ml_engineer_skills-0.4.0/src/mlcheck/__main__.py +5 -0
  14. ml_engineer_skills-0.4.0/src/mlcheck/baselines.py +124 -0
  15. ml_engineer_skills-0.4.0/src/mlcheck/cli.py +293 -0
  16. ml_engineer_skills-0.4.0/src/mlcheck/core.py +153 -0
  17. ml_engineer_skills-0.4.0/src/mlcheck/detectors.py +192 -0
  18. ml_engineer_skills-0.4.0/src/mlcheck/fixes.py +57 -0
  19. ml_engineer_skills-0.4.0/src/mlcheck/impact.py +73 -0
  20. ml_engineer_skills-0.4.0/src/mlcheck/integrations/__init__.py +7 -0
  21. ml_engineer_skills-0.4.0/src/mlcheck/integrations/jupyter.py +41 -0
  22. ml_engineer_skills-0.4.0/src/mlcheck/integrations/pytest_plugin.py +79 -0
  23. ml_engineer_skills-0.4.0/src/mlcheck/leakage.py +201 -0
  24. ml_engineer_skills-0.4.0/src/mlcheck/metrics.py +82 -0
  25. ml_engineer_skills-0.4.0/src/mlcheck/repro.py +120 -0
  26. ml_engineer_skills-0.4.0/src/mlcheck/splits.py +120 -0
  27. ml_engineer_skills-0.4.0/tests/test_baselines_metrics_extra.py +37 -0
  28. ml_engineer_skills-0.4.0/tests/test_choosing_evaluation_metrics.py +26 -0
  29. ml_engineer_skills-0.4.0/tests/test_cli.py +54 -0
  30. ml_engineer_skills-0.4.0/tests/test_cli_v03.py +48 -0
  31. ml_engineer_skills-0.4.0/tests/test_core.py +28 -0
  32. ml_engineer_skills-0.4.0/tests/test_designing_validation_splits.py +33 -0
  33. ml_engineer_skills-0.4.0/tests/test_detecting_data_leakage.py +42 -0
  34. ml_engineer_skills-0.4.0/tests/test_detectors.py +53 -0
  35. ml_engineer_skills-0.4.0/tests/test_establishing_baselines.py +32 -0
  36. ml_engineer_skills-0.4.0/tests/test_impact.py +23 -0
  37. ml_engineer_skills-0.4.0/tests/test_integrations.py +51 -0
  38. ml_engineer_skills-0.4.0/tests/test_leakage_patterns.py +46 -0
  39. ml_engineer_skills-0.4.0/tests/test_output_and_fixes.py +61 -0
  40. ml_engineer_skills-0.4.0/tests/test_repro.py +29 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Krishi Shah
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,167 @@
1
+ Metadata-Version: 2.4
2
+ Name: ml-engineer-skills
3
+ Version: 0.4.0
4
+ Summary: A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command.
5
+ Author: Krishi Shah
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/krishi-shah/ml-engineer-skills
8
+ Project-URL: Repository, https://github.com/krishi-shah/ml-engineer-skills
9
+ Project-URL: Issues, https://github.com/krishi-shah/ml-engineer-skills/issues
10
+ Keywords: machine-learning,ml-engineering,data-leakage,model-validation,mlops,scikit-learn
11
+ Requires-Python: >=3.9
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: numpy>=1.24
15
+ Requires-Dist: pandas>=2.0
16
+ Requires-Dist: scikit-learn>=1.3
17
+ Provides-Extra: dev
18
+ Requires-Dist: pytest>=7.4; extra == "dev"
19
+ Provides-Extra: notebook
20
+ Requires-Dist: ipython; extra == "notebook"
21
+ Dynamic: license-file
22
+
23
+ # ML Engineer Skills
24
+
25
+ [![CI](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml/badge.svg)](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
26
+ [![PyPI](https://img.shields.io/pypi/v/ml-engineer-skills)](https://pypi.org/project/ml-engineer-skills/)
27
+ [![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
28
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
29
+
30
+ **A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
31
+
32
+ <!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
33
+ <!-- ![mlcheck catching five bugs in one command](docs/demo.gif) -->
34
+
35
+ ## Why you'd want this
36
+
37
+ When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
38
+
39
+ `mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
40
+
41
+ ## Install
42
+
43
+ ```bash
44
+ pip install ml-engineer-skills
45
+ ```
46
+
47
+ That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
48
+
49
+ ## Try it in 30 seconds
50
+
51
+ ```bash
52
+ python example/quickstart.py
53
+ ```
54
+
55
+ You'll see it catch 5 real bugs at once, each with a fix:
56
+
57
+ ```
58
+ [x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
59
+ -> fix: Move the transform into a Pipeline fit AFTER the split.
60
+ [x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
61
+ [x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
62
+ ...
63
+ ```
64
+
65
+ ## Use it on your own project
66
+
67
+ Check a training script for leaks:
68
+
69
+ ```bash
70
+ mlcheck scan-source train.py
71
+ ```
72
+
73
+ Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
74
+
75
+ ```bash
76
+ mlcheck audit --source train.py --train train.csv --test test.csv \
77
+ --target label --fail-on error
78
+ ```
79
+
80
+ Want the corrected code, not just the warning? Add `--suggest`:
81
+
82
+ ```bash
83
+ mlcheck scan-source train.py --suggest
84
+ ```
85
+
86
+ ## What it checks for
87
+
88
+ | Mistake | What it means (plain English) |
89
+ |---|---|
90
+ | **Data leakage** | Your model secretly saw information it won't have in real life. |
91
+ | **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
92
+ | **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
93
+ | **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
94
+ | **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
95
+ | **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
96
+
97
+ ## Know your *real* score
98
+
99
+ This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
100
+
101
+ ```python
102
+ from mlcheck import split_impact
103
+ for finding in split_impact(model, X, y, group_col="user_id"):
104
+ print(finding)
105
+ # Naive split scores 1.000, but an honest split scores 0.733.
106
+ # Your real number is about 0.733.
107
+ ```
108
+
109
+ That honest number is the one that holds up in the real world.
110
+
111
+ ## Run it automatically (so you never forget)
112
+
113
+ **In your tests** — fail the build if there's a leak:
114
+ ```python
115
+ from mlcheck.integrations.pytest_plugin import assert_no_leakage
116
+ def test_no_leakage():
117
+ assert_no_leakage("train.py")
118
+ ```
119
+
120
+ **Before every commit** — add to `.pre-commit-config.yaml`:
121
+ ```yaml
122
+ - repo: https://github.com/krishi-shah/ml-engineer-skills
123
+ rev: v0.4.0
124
+ hooks:
125
+ - id: mlcheck-scan-source
126
+ ```
127
+
128
+ **In a Jupyter notebook** — flag leaks as you write them:
129
+ ```python
130
+ %load_ext mlcheck.integrations.jupyter
131
+ %%mlcheck
132
+ scaler = StandardScaler().fit(X) # flagged right after the cell runs
133
+ ```
134
+
135
+ **With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
136
+ ```
137
+ /plugin marketplace add krishi-shah/ml-engineer-skills
138
+ /plugin install ml-engineer-skills
139
+ ```
140
+
141
+ ## Is it for me?
142
+
143
+ - **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
144
+ - **On a team?** Yes — wire it into CI so a leaked model can't get merged.
145
+ - **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
146
+
147
+ It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
148
+
149
+ ## What's inside
150
+
151
+ ```
152
+ src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
153
+ skills/ the playbooks that teach each check (and guide an AI assistant)
154
+ example/ quickstart.py + a deliberately-buggy script to scan
155
+ tests/ 64 tests, each proving a check catches a real bug
156
+ ```
157
+
158
+ Verify it yourself:
159
+
160
+ ```bash
161
+ pip install -e ".[dev]"
162
+ pytest -q # 64 passed
163
+ ```
164
+
165
+ ## License
166
+
167
+ MIT
@@ -0,0 +1,145 @@
1
+ # ML Engineer Skills
2
+
3
+ [![CI](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml/badge.svg)](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/ml-engineer-skills)](https://pypi.org/project/ml-engineer-skills/)
5
+ [![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
6
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
7
+
8
+ **A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
9
+
10
+ <!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
11
+ <!-- ![mlcheck catching five bugs in one command](docs/demo.gif) -->
12
+
13
+ ## Why you'd want this
14
+
15
+ When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
16
+
17
+ `mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
18
+
19
+ ## Install
20
+
21
+ ```bash
22
+ pip install ml-engineer-skills
23
+ ```
24
+
25
+ That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
26
+
27
+ ## Try it in 30 seconds
28
+
29
+ ```bash
30
+ python example/quickstart.py
31
+ ```
32
+
33
+ You'll see it catch 5 real bugs at once, each with a fix:
34
+
35
+ ```
36
+ [x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
37
+ -> fix: Move the transform into a Pipeline fit AFTER the split.
38
+ [x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
39
+ [x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
40
+ ...
41
+ ```
42
+
43
+ ## Use it on your own project
44
+
45
+ Check a training script for leaks:
46
+
47
+ ```bash
48
+ mlcheck scan-source train.py
49
+ ```
50
+
51
+ Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
52
+
53
+ ```bash
54
+ mlcheck audit --source train.py --train train.csv --test test.csv \
55
+ --target label --fail-on error
56
+ ```
57
+
58
+ Want the corrected code, not just the warning? Add `--suggest`:
59
+
60
+ ```bash
61
+ mlcheck scan-source train.py --suggest
62
+ ```
63
+
64
+ ## What it checks for
65
+
66
+ | Mistake | What it means (plain English) |
67
+ |---|---|
68
+ | **Data leakage** | Your model secretly saw information it won't have in real life. |
69
+ | **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
70
+ | **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
71
+ | **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
72
+ | **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
73
+ | **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
74
+
75
+ ## Know your *real* score
76
+
77
+ This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
78
+
79
+ ```python
80
+ from mlcheck import split_impact
81
+ for finding in split_impact(model, X, y, group_col="user_id"):
82
+ print(finding)
83
+ # Naive split scores 1.000, but an honest split scores 0.733.
84
+ # Your real number is about 0.733.
85
+ ```
86
+
87
+ That honest number is the one that holds up in the real world.
88
+
89
+ ## Run it automatically (so you never forget)
90
+
91
+ **In your tests** — fail the build if there's a leak:
92
+ ```python
93
+ from mlcheck.integrations.pytest_plugin import assert_no_leakage
94
+ def test_no_leakage():
95
+ assert_no_leakage("train.py")
96
+ ```
97
+
98
+ **Before every commit** — add to `.pre-commit-config.yaml`:
99
+ ```yaml
100
+ - repo: https://github.com/krishi-shah/ml-engineer-skills
101
+ rev: v0.4.0
102
+ hooks:
103
+ - id: mlcheck-scan-source
104
+ ```
105
+
106
+ **In a Jupyter notebook** — flag leaks as you write them:
107
+ ```python
108
+ %load_ext mlcheck.integrations.jupyter
109
+ %%mlcheck
110
+ scaler = StandardScaler().fit(X) # flagged right after the cell runs
111
+ ```
112
+
113
+ **With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
114
+ ```
115
+ /plugin marketplace add krishi-shah/ml-engineer-skills
116
+ /plugin install ml-engineer-skills
117
+ ```
118
+
119
+ ## Is it for me?
120
+
121
+ - **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
122
+ - **On a team?** Yes — wire it into CI so a leaked model can't get merged.
123
+ - **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
124
+
125
+ It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
126
+
127
+ ## What's inside
128
+
129
+ ```
130
+ src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
131
+ skills/ the playbooks that teach each check (and guide an AI assistant)
132
+ example/ quickstart.py + a deliberately-buggy script to scan
133
+ tests/ 64 tests, each proving a check catches a real bug
134
+ ```
135
+
136
+ Verify it yourself:
137
+
138
+ ```bash
139
+ pip install -e ".[dev]"
140
+ pytest -q # 64 passed
141
+ ```
142
+
143
+ ## License
144
+
145
+ MIT
@@ -0,0 +1,40 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "ml-engineer-skills"
7
+ version = "0.4.0"
8
+ description = "A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Krishi Shah" }]
13
+ keywords = ["machine-learning", "ml-engineering", "data-leakage", "model-validation", "mlops", "scikit-learn"]
14
+ dependencies = [
15
+ "numpy>=1.24",
16
+ "pandas>=2.0",
17
+ "scikit-learn>=1.3",
18
+ ]
19
+
20
+ [project.urls]
21
+ Homepage = "https://github.com/krishi-shah/ml-engineer-skills"
22
+ Repository = "https://github.com/krishi-shah/ml-engineer-skills"
23
+ Issues = "https://github.com/krishi-shah/ml-engineer-skills/issues"
24
+
25
+ [project.optional-dependencies]
26
+ dev = ["pytest>=7.4"]
27
+ notebook = ["ipython"]
28
+
29
+ [project.scripts]
30
+ mlcheck = "mlcheck.cli:main"
31
+
32
+ [project.entry-points.pytest11]
33
+ mlcheck = "mlcheck.integrations.pytest_plugin"
34
+
35
+ [tool.setuptools.packages.find]
36
+ where = ["src"]
37
+
38
+ [tool.pytest.ini_options]
39
+ pythonpath = ["src"]
40
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,167 @@
1
+ Metadata-Version: 2.4
2
+ Name: ml-engineer-skills
3
+ Version: 0.4.0
4
+ Summary: A spell-checker for ML mistakes: catches data leakage, broken splits, unbeaten baselines, and metric misuse. Provides the `mlcheck` command.
5
+ Author: Krishi Shah
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/krishi-shah/ml-engineer-skills
8
+ Project-URL: Repository, https://github.com/krishi-shah/ml-engineer-skills
9
+ Project-URL: Issues, https://github.com/krishi-shah/ml-engineer-skills/issues
10
+ Keywords: machine-learning,ml-engineering,data-leakage,model-validation,mlops,scikit-learn
11
+ Requires-Python: >=3.9
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: numpy>=1.24
15
+ Requires-Dist: pandas>=2.0
16
+ Requires-Dist: scikit-learn>=1.3
17
+ Provides-Extra: dev
18
+ Requires-Dist: pytest>=7.4; extra == "dev"
19
+ Provides-Extra: notebook
20
+ Requires-Dist: ipython; extra == "notebook"
21
+ Dynamic: license-file
22
+
23
+ # ML Engineer Skills
24
+
25
+ [![CI](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml/badge.svg)](https://github.com/krishi-shah/ml-engineer-skills/actions/workflows/ci.yml)
26
+ [![PyPI](https://img.shields.io/pypi/v/ml-engineer-skills)](https://pypi.org/project/ml-engineer-skills/)
27
+ [![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
28
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
29
+
30
+ **A spell-checker for machine-learning mistakes.** Point it at your code and data, and it tells you — in plain English — what's wrong and how to fix it. The tool you install and run is called **`mlcheck`**.
31
+
32
+ <!-- To add the demo GIF: run `vhs demo/demo.tape` (see demo/README), commit docs/demo.gif, then uncomment the line below. -->
33
+ <!-- ![mlcheck catching five bugs in one command](docs/demo.gif) -->
34
+
35
+ ## Why you'd want this
36
+
37
+ When you train a model, you get a score like *"98% accurate!"*. The problem: a score that looks amazing is very often **fake** — not because you cheated, but because of small, invisible mistakes almost everyone makes (your model accidentally saw the answers, or you tested it on data it already studied). The model looks brilliant in your notebook, then falls apart on real data.
38
+
39
+ `mlcheck` catches those mistakes early, explains them, and even shows you your model's **real** score.
40
+
41
+ ## Install
42
+
43
+ ```bash
44
+ pip install ml-engineer-skills
45
+ ```
46
+
47
+ That gives you the **`mlcheck`** command. (Prefer the latest from source? `pip install git+https://github.com/krishi-shah/ml-engineer-skills.git`)
48
+
49
+ ## Try it in 30 seconds
50
+
51
+ ```bash
52
+ python example/quickstart.py
53
+ ```
54
+
55
+ You'll see it catch 5 real bugs at once, each with a fix:
56
+
57
+ ```
58
+ [x] FIT_BEFORE_SPLIT (line 26): scaling happened before the train/test split.
59
+ -> fix: Move the transform into a Pipeline fit AFTER the split.
60
+ [x] NO_LIFT: accuracy 0.99 does NOT beat the dumb baseline (0.99) - the model learned nothing.
61
+ [x] ACCURACY_ON_IMBALANCE: 99% of rows are one class, so "accuracy" is meaningless here.
62
+ ...
63
+ ```
64
+
65
+ ## Use it on your own project
66
+
67
+ Check a training script for leaks:
68
+
69
+ ```bash
70
+ mlcheck scan-source train.py
71
+ ```
72
+
73
+ Run every check at once and get one report (great for CI — it exits with an error if something's wrong):
74
+
75
+ ```bash
76
+ mlcheck audit --source train.py --train train.csv --test test.csv \
77
+ --target label --fail-on error
78
+ ```
79
+
80
+ Want the corrected code, not just the warning? Add `--suggest`:
81
+
82
+ ```bash
83
+ mlcheck scan-source train.py --suggest
84
+ ```
85
+
86
+ ## What it checks for
87
+
88
+ | Mistake | What it means (plain English) |
89
+ |---|---|
90
+ | **Data leakage** | Your model secretly saw information it won't have in real life. |
91
+ | **Bad train/test split** | You tested the model on data it already studied, so the score is inflated. |
92
+ | **No baseline** | "98%" sounds great — until you learn that *always guessing the same answer* also scores 98%. |
93
+ | **Wrong metric** | You're measuring the wrong thing (e.g. "accuracy" on rare-event data). |
94
+ | **Not reproducible** | The score changes every run because randomness wasn't pinned down. |
95
+ | **Broken training** | The model can't even memorize a few examples — a wiring bug, not bad luck. |
96
+
97
+ ## Know your *real* score
98
+
99
+ This is the part that actually helps your accuracy. Instead of just saying "you have a leak," it measures what the leak cost you:
100
+
101
+ ```python
102
+ from mlcheck import split_impact
103
+ for finding in split_impact(model, X, y, group_col="user_id"):
104
+ print(finding)
105
+ # Naive split scores 1.000, but an honest split scores 0.733.
106
+ # Your real number is about 0.733.
107
+ ```
108
+
109
+ That honest number is the one that holds up in the real world.
110
+
111
+ ## Run it automatically (so you never forget)
112
+
113
+ **In your tests** — fail the build if there's a leak:
114
+ ```python
115
+ from mlcheck.integrations.pytest_plugin import assert_no_leakage
116
+ def test_no_leakage():
117
+ assert_no_leakage("train.py")
118
+ ```
119
+
120
+ **Before every commit** — add to `.pre-commit-config.yaml`:
121
+ ```yaml
122
+ - repo: https://github.com/krishi-shah/ml-engineer-skills
123
+ rev: v0.4.0
124
+ hooks:
125
+ - id: mlcheck-scan-source
126
+ ```
127
+
128
+ **In a Jupyter notebook** — flag leaks as you write them:
129
+ ```python
130
+ %load_ext mlcheck.integrations.jupyter
131
+ %%mlcheck
132
+ scaler = StandardScaler().fit(X) # flagged right after the cell runs
133
+ ```
134
+
135
+ **With an AI assistant (Claude Code)** — install the skills so your assistant checks its own ML work:
136
+ ```
137
+ /plugin marketplace add krishi-shah/ml-engineer-skills
138
+ /plugin install ml-engineer-skills
139
+ ```
140
+
141
+ ## Is it for me?
142
+
143
+ - **New to ML / a student?** Yes — it catches the exact mistakes you don't yet know to look for, and teaches you as it goes.
144
+ - **On a team?** Yes — wire it into CI so a leaked model can't get merged.
145
+ - **Already an expert?** You'll get less out of it; you've built these habits. It's still a useful safety net.
146
+
147
+ It's a smoke detector, not a guarantee: it catches the *common* mistakes, which is most of them.
148
+
149
+ ## What's inside
150
+
151
+ ```
152
+ src/mlcheck/ the tool (plain Python: numpy, pandas, scikit-learn)
153
+ skills/ the playbooks that teach each check (and guide an AI assistant)
154
+ example/ quickstart.py + a deliberately-buggy script to scan
155
+ tests/ 64 tests, each proving a check catches a real bug
156
+ ```
157
+
158
+ Verify it yourself:
159
+
160
+ ```bash
161
+ pip install -e ".[dev]"
162
+ pytest -q # 64 passed
163
+ ```
164
+
165
+ ## License
166
+
167
+ MIT
@@ -0,0 +1,38 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ src/ml_engineer_skills.egg-info/PKG-INFO
5
+ src/ml_engineer_skills.egg-info/SOURCES.txt
6
+ src/ml_engineer_skills.egg-info/dependency_links.txt
7
+ src/ml_engineer_skills.egg-info/entry_points.txt
8
+ src/ml_engineer_skills.egg-info/requires.txt
9
+ src/ml_engineer_skills.egg-info/top_level.txt
10
+ src/mlcheck/__init__.py
11
+ src/mlcheck/__main__.py
12
+ src/mlcheck/baselines.py
13
+ src/mlcheck/cli.py
14
+ src/mlcheck/core.py
15
+ src/mlcheck/detectors.py
16
+ src/mlcheck/fixes.py
17
+ src/mlcheck/impact.py
18
+ src/mlcheck/leakage.py
19
+ src/mlcheck/metrics.py
20
+ src/mlcheck/repro.py
21
+ src/mlcheck/splits.py
22
+ src/mlcheck/integrations/__init__.py
23
+ src/mlcheck/integrations/jupyter.py
24
+ src/mlcheck/integrations/pytest_plugin.py
25
+ tests/test_baselines_metrics_extra.py
26
+ tests/test_choosing_evaluation_metrics.py
27
+ tests/test_cli.py
28
+ tests/test_cli_v03.py
29
+ tests/test_core.py
30
+ tests/test_designing_validation_splits.py
31
+ tests/test_detecting_data_leakage.py
32
+ tests/test_detectors.py
33
+ tests/test_establishing_baselines.py
34
+ tests/test_impact.py
35
+ tests/test_integrations.py
36
+ tests/test_leakage_patterns.py
37
+ tests/test_output_and_fixes.py
38
+ tests/test_repro.py
@@ -0,0 +1,5 @@
1
+ [console_scripts]
2
+ mlcheck = mlcheck.cli:main
3
+
4
+ [pytest11]
5
+ mlcheck = mlcheck.integrations.pytest_plugin
@@ -0,0 +1,9 @@
1
+ numpy>=1.24
2
+ pandas>=2.0
3
+ scikit-learn>=1.3
4
+
5
+ [dev]
6
+ pytest>=7.4
7
+
8
+ [notebook]
9
+ ipython
@@ -0,0 +1,52 @@
1
+ """mlcheck — the ML rigor toolkit.
2
+
3
+ Runnable diagnostics that catch the silent mistakes between "works in the
4
+ notebook" and "works in production": unbeaten baselines, broken validation
5
+ splits, data leakage, and metric malpractice.
6
+
7
+ Public API:
8
+ from mlcheck import (
9
+ Finding, Severity, format_report, worst_severity,
10
+ evaluate_baseline, audit_split, scan_source, scan_data, advise_metric,
11
+ )
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from .core import Finding, Severity, format_report, format_json, format_markdown, worst_severity
16
+ from .baselines import evaluate_baseline, baseline_score
17
+ from .splits import audit_split
18
+ from .leakage import scan_source, scan_data
19
+ from .metrics import advise_metric
20
+ from .detectors import (
21
+ adversarial_validation,
22
+ signal_is_real,
23
+ target_leak_scan,
24
+ can_overfit_small_sample,
25
+ )
26
+ from .repro import scan_reproducibility, data_fingerprint
27
+ from .impact import split_impact
28
+
29
+ __version__ = "0.4.0"
30
+
31
+ __all__ = [
32
+ "Finding",
33
+ "Severity",
34
+ "format_report",
35
+ "format_json",
36
+ "format_markdown",
37
+ "worst_severity",
38
+ "split_impact",
39
+ "evaluate_baseline",
40
+ "baseline_score",
41
+ "audit_split",
42
+ "scan_source",
43
+ "scan_data",
44
+ "advise_metric",
45
+ "adversarial_validation",
46
+ "signal_is_real",
47
+ "target_leak_scan",
48
+ "can_overfit_small_sample",
49
+ "scan_reproducibility",
50
+ "data_fingerprint",
51
+ "__version__",
52
+ ]
@@ -0,0 +1,5 @@
1
+ """Enable `python -m mlcheck ...`."""
2
+ from .cli import main
3
+
4
+ if __name__ == "__main__":
5
+ raise SystemExit(main())