trainlint 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trainlint-0.1.0/.claude/settings.local.json +7 -0
- trainlint-0.1.0/.gitattributes +1 -0
- trainlint-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +23 -0
- trainlint-0.1.0/.github/ISSUE_TEMPLATE/config.yml +1 -0
- trainlint-0.1.0/.github/ISSUE_TEMPLATE/false_positive.yml +30 -0
- trainlint-0.1.0/.github/ISSUE_TEMPLATE/rule_idea.yml +27 -0
- trainlint-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +8 -0
- trainlint-0.1.0/.github/workflows/ci.yml +43 -0
- trainlint-0.1.0/.gitignore +8 -0
- trainlint-0.1.0/CHANGELOG.md +14 -0
- trainlint-0.1.0/CONTRIBUTING.md +46 -0
- trainlint-0.1.0/LICENSE +21 -0
- trainlint-0.1.0/PKG-INFO +204 -0
- trainlint-0.1.0/README.md +183 -0
- trainlint-0.1.0/assets/demo.gif +0 -0
- trainlint-0.1.0/assets/make_assets.py +187 -0
- trainlint-0.1.0/assets/social-preview.png +0 -0
- trainlint-0.1.0/bench/__init__.py +0 -0
- trainlint-0.1.0/bench/__main__.py +25 -0
- trainlint-0.1.0/bench/buggy/tl001_scaler_before_split.py +11 -0
- trainlint-0.1.0/bench/buggy/tl001_select_k_best_cv.py +11 -0
- trainlint-0.1.0/bench/buggy/tl001_smote_before_split.py +11 -0
- trainlint-0.1.0/bench/buggy/tl002_scaler_fit_on_test.py +13 -0
- trainlint-0.1.0/bench/buggy/tl003_eval_left_on.py +28 -0
- trainlint-0.1.0/bench/buggy/tl004_no_grad_without_eval.py +31 -0
- trainlint-0.1.0/bench/buggy/tl005_no_seed.py +31 -0
- trainlint-0.1.0/bench/buggy/tl006_fp16_without_scaler.py +33 -0
- trainlint-0.1.0/bench/buggy/tl006_scaled_loss_plain_step.py +35 -0
- trainlint-0.1.0/bench/buggy/tl007_resume_without_optimizer_state.py +33 -0
- trainlint-0.1.0/bench/buggy/tl008_sigmoid_before_bce_with_logits.py +17 -0
- trainlint-0.1.0/bench/buggy/tl008_softmax_before_cross_entropy.py +41 -0
- trainlint-0.1.0/bench/buggy/tl009_gradients_never_cleared.py +31 -0
- trainlint-0.1.0/bench/buggy/tl009_zero_grad_after_backward.py +32 -0
- trainlint-0.1.0/bench/buggy/tl010_running_loss_tensor.py +34 -0
- trainlint-0.1.0/bench/buggy/tl011_auc_from_class_labels.py +11 -0
- trainlint-0.1.0/bench/buggy/tl011_swapped_metric_arguments.py +11 -0
- trainlint-0.1.0/bench/buggy/tl012_validation_loader_drops_last.py +32 -0
- trainlint-0.1.0/bench/corpus.py +64 -0
- trainlint-0.1.0/bench/fixed/tl001_scaler_before_split.py +10 -0
- trainlint-0.1.0/bench/fixed/tl001_select_k_best_cv.py +10 -0
- trainlint-0.1.0/bench/fixed/tl001_smote_before_split.py +10 -0
- trainlint-0.1.0/bench/fixed/tl002_scaler_fit_on_test.py +12 -0
- trainlint-0.1.0/bench/fixed/tl003_eval_left_on.py +27 -0
- trainlint-0.1.0/bench/fixed/tl004_no_grad_without_eval.py +31 -0
- trainlint-0.1.0/bench/fixed/tl005_no_seed.py +31 -0
- trainlint-0.1.0/bench/fixed/tl006_fp16_without_scaler.py +34 -0
- trainlint-0.1.0/bench/fixed/tl006_scaled_loss_plain_step.py +34 -0
- trainlint-0.1.0/bench/fixed/tl007_resume_without_optimizer_state.py +39 -0
- trainlint-0.1.0/bench/fixed/tl008_sigmoid_before_bce_with_logits.py +16 -0
- trainlint-0.1.0/bench/fixed/tl008_softmax_before_cross_entropy.py +40 -0
- trainlint-0.1.0/bench/fixed/tl009_gradients_never_cleared.py +31 -0
- trainlint-0.1.0/bench/fixed/tl009_zero_grad_after_backward.py +31 -0
- trainlint-0.1.0/bench/fixed/tl010_running_loss_tensor.py +33 -0
- trainlint-0.1.0/bench/fixed/tl011_auc_from_class_labels.py +10 -0
- trainlint-0.1.0/bench/fixed/tl011_swapped_metric_arguments.py +10 -0
- trainlint-0.1.0/bench/fixed/tl012_validation_loader_drops_last.py +31 -0
- trainlint-0.1.0/docs/rules.md +250 -0
- trainlint-0.1.0/examples/noise_leak.py +21 -0
- trainlint-0.1.0/examples/noise_leak_fixed.py +18 -0
- trainlint-0.1.0/notes/publish.sh +113 -0
- trainlint-0.1.0/notes/release-notes.md +15 -0
- trainlint-0.1.0/notes/spec.md +50 -0
- trainlint-0.1.0/pyproject.toml +53 -0
- trainlint-0.1.0/src/trainlint/__init__.py +3 -0
- trainlint-0.1.0/src/trainlint/__main__.py +3 -0
- trainlint-0.1.0/src/trainlint/astutil.py +122 -0
- trainlint-0.1.0/src/trainlint/cli.py +118 -0
- trainlint-0.1.0/src/trainlint/engine.py +132 -0
- trainlint-0.1.0/src/trainlint/report.py +72 -0
- trainlint-0.1.0/src/trainlint/rules/__init__.py +19 -0
- trainlint-0.1.0/src/trainlint/rules/base.py +21 -0
- trainlint-0.1.0/src/trainlint/rules/checkpoints.py +68 -0
- trainlint-0.1.0/src/trainlint/rules/evaluation.py +117 -0
- trainlint-0.1.0/src/trainlint/rules/leakage.py +115 -0
- trainlint-0.1.0/src/trainlint/rules/mixed_precision.py +85 -0
- trainlint-0.1.0/src/trainlint/rules/training.py +302 -0
- trainlint-0.1.0/src/trainlint/skills/ml-checkpoint-resume/SKILL.md +70 -0
- trainlint-0.1.0/src/trainlint/skills/ml-data-leakage/SKILL.md +73 -0
- trainlint-0.1.0/src/trainlint/skills/ml-metrics-eval/SKILL.md +53 -0
- trainlint-0.1.0/src/trainlint/skills/ml-training-loop/SKILL.md +98 -0
- trainlint-0.1.0/src/trainlint/skills_install.py +36 -0
- trainlint-0.1.0/tests/conftest.py +17 -0
- trainlint-0.1.0/tests/test_bench.py +27 -0
- trainlint-0.1.0/tests/test_cli.py +72 -0
- trainlint-0.1.0/tests/test_docs.py +38 -0
- trainlint-0.1.0/tests/test_engine.py +64 -0
- trainlint-0.1.0/tests/test_leakage.py +135 -0
- trainlint-0.1.0/tests/test_precision_checkpoints_metrics.py +188 -0
- trainlint-0.1.0/tests/test_skills.py +65 -0
- trainlint-0.1.0/tests/test_training.py +322 -0
- trainlint-0.1.0/uv.lock +202 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* text=auto eol=lf
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Something other than a wrong finding is broken
|
|
3
|
+
labels: ["bug"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: what
|
|
7
|
+
attributes:
|
|
8
|
+
label: What happened
|
|
9
|
+
description: The command you ran, what you expected, and what you got.
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: input
|
|
13
|
+
id: version
|
|
14
|
+
attributes:
|
|
15
|
+
label: Output of `trainlint --version`
|
|
16
|
+
validations:
|
|
17
|
+
required: true
|
|
18
|
+
- type: input
|
|
19
|
+
id: platform
|
|
20
|
+
attributes:
|
|
21
|
+
label: Operating system and Python version
|
|
22
|
+
validations:
|
|
23
|
+
required: true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
blank_issues_enabled: false
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: False positive
|
|
2
|
+
description: trainlint flagged code that is correct
|
|
3
|
+
labels: ["false positive"]
|
|
4
|
+
body:
|
|
5
|
+
- type: input
|
|
6
|
+
id: rule
|
|
7
|
+
attributes:
|
|
8
|
+
label: Rule
|
|
9
|
+
placeholder: TL001
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: snippet
|
|
14
|
+
attributes:
|
|
15
|
+
label: Smallest snippet that is flagged
|
|
16
|
+
render: python
|
|
17
|
+
validations:
|
|
18
|
+
required: true
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: why
|
|
21
|
+
attributes:
|
|
22
|
+
label: Why this code is correct
|
|
23
|
+
validations:
|
|
24
|
+
required: true
|
|
25
|
+
- type: input
|
|
26
|
+
id: version
|
|
27
|
+
attributes:
|
|
28
|
+
label: Output of `trainlint --version`
|
|
29
|
+
validations:
|
|
30
|
+
required: true
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Rule idea
|
|
2
|
+
description: A bug that trainlint should catch
|
|
3
|
+
labels: ["new rule"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: bug
|
|
7
|
+
attributes:
|
|
8
|
+
label: The bug
|
|
9
|
+
description: What goes wrong, and how you would notice it (or not).
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: example
|
|
14
|
+
attributes:
|
|
15
|
+
label: Minimal example
|
|
16
|
+
render: python
|
|
17
|
+
validations:
|
|
18
|
+
required: true
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: signal
|
|
21
|
+
attributes:
|
|
22
|
+
label: What in the code gives it away
|
|
23
|
+
description: |
|
|
24
|
+
Rules work on one file's syntax tree, with no type information. Name the calls, names or
|
|
25
|
+
ordering a check could look at, and a sensible script that looks similar but is fine.
|
|
26
|
+
validations:
|
|
27
|
+
required: true
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
## What changed
|
|
2
|
+
|
|
3
|
+
## How I checked it
|
|
4
|
+
|
|
5
|
+
- [ ] `uv run pytest` passes
|
|
6
|
+
- [ ] `uv run ruff check . && uv run ruff format --check .` passes
|
|
7
|
+
- [ ] New or changed rule: buggy and fixed files added under `bench/`, example added to `docs/rules.md`
|
|
8
|
+
- [ ] New or changed rule: I read what it reports on at least one real codebase
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
name: test (${{ matrix.os }}, python ${{ matrix.python }})
|
|
11
|
+
runs-on: ${{ matrix.os }}
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
16
|
+
python: ["3.10", "3.13"]
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: astral-sh/setup-uv@v5
|
|
20
|
+
- run: uv sync --locked --python ${{ matrix.python }}
|
|
21
|
+
- run: uv run pytest -q
|
|
22
|
+
- run: uv run python -m bench
|
|
23
|
+
|
|
24
|
+
lint:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v4
|
|
28
|
+
- uses: astral-sh/setup-uv@v5
|
|
29
|
+
- run: uv sync --locked
|
|
30
|
+
- run: uv run ruff check .
|
|
31
|
+
- run: uv run ruff format --check .
|
|
32
|
+
|
|
33
|
+
package:
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@v4
|
|
37
|
+
- uses: astral-sh/setup-uv@v5
|
|
38
|
+
- run: uv build
|
|
39
|
+
- name: Install the wheel into a clean environment and run it
|
|
40
|
+
run: |
|
|
41
|
+
uvx --from dist/*.whl trainlint skills install --dest "$RUNNER_TEMP/skills"
|
|
42
|
+
test -f "$RUNNER_TEMP/skills/ml-training-loop/SKILL.md"
|
|
43
|
+
uvx --from dist/*.whl trainlint check --exit-zero bench/buggy
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 - 2026-11-03
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- `trainlint check` with 12 rules (TL001 to TL012) for data leakage, train and eval mode,
|
|
8
|
+
seeding, mixed precision, checkpoints, loss inputs, gradient clearing, tensor accumulation,
|
|
9
|
+
metric misuse and evaluation loaders. Text and JSON output, `--select`, `--ignore`,
|
|
10
|
+
`--exit-zero`, `--color`, and `# trainlint: ignore[RULE]` comments.
|
|
11
|
+
- Four Agent Skills (`ml-data-leakage`, `ml-training-loop`, `ml-metrics-eval`,
|
|
12
|
+
`ml-checkpoint-resume`) and `trainlint skills install`.
|
|
13
|
+
- Regression corpus of 18 buggy scripts with fixed twins (`python -m bench`).
|
|
14
|
+
- Per-rule reference in `docs/rules.md`.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Reports of false positives and missed bugs are the most useful thing you can send. A rule that
|
|
4
|
+
cries wolf gets switched off, so precision matters more here than coverage.
|
|
5
|
+
|
|
6
|
+
## Set up
|
|
7
|
+
|
|
8
|
+
You need [uv](https://docs.astral.sh/uv/) and Python 3.10 or newer.
|
|
9
|
+
|
|
10
|
+
```console
|
|
11
|
+
git clone https://github.com/Satvik-jain/trainlint
|
|
12
|
+
cd trainlint
|
|
13
|
+
uv sync
|
|
14
|
+
uv run pytest
|
|
15
|
+
uv run ruff check . && uv run ruff format --check .
|
|
16
|
+
uv run python -m bench
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Reporting a problem
|
|
20
|
+
|
|
21
|
+
- **False positive:** use the "False positive" issue form. Paste the smallest snippet that is
|
|
22
|
+
flagged and say why it is fine.
|
|
23
|
+
- **Missed bug:** use the "Rule idea" form. Show the bug and say what in the code gives it away.
|
|
24
|
+
A rule has to work from one file's syntax tree, with no types and no imports resolved.
|
|
25
|
+
|
|
26
|
+
## Adding or changing a rule
|
|
27
|
+
|
|
28
|
+
1. Write the check as a function `(tree: ast.Module) -> Iterator[Hit]` in the module for its
|
|
29
|
+
topic under `src/trainlint/rules/`, and register it in that module's `RULES`. Helpers for
|
|
30
|
+
matching calls and names live in `src/trainlint/astutil.py`.
|
|
31
|
+
2. Add unit tests with both flagged and clean snippets. Clean snippets matter more: write down
|
|
32
|
+
the ways a sensible script could look like the bug without being it.
|
|
33
|
+
3. Add a buggy script and its fixed twin to `bench/buggy/` and `bench/fixed/`. The buggy file
|
|
34
|
+
starts with `# expect: TLxxx`.
|
|
35
|
+
4. Add the rule to `docs/rules.md` with one flagged and one clean example. A test runs both.
|
|
36
|
+
5. If a bundled skill should mention it, add it to the skill under `src/trainlint/skills/`.
|
|
37
|
+
A test checks that skills only cite rules that exist and that every rule is cited.
|
|
38
|
+
|
|
39
|
+
A new rule should be quiet on ordinary code. If you can, run it over a few real repositories and
|
|
40
|
+
read what it reports before opening the pull request.
|
|
41
|
+
|
|
42
|
+
## Commits and pull requests
|
|
43
|
+
|
|
44
|
+
Keep commits small and use Conventional Commit prefixes (`feat`, `fix`, `docs`, `test`, `chore`,
|
|
45
|
+
`refactor`, `ci`), imperative mood, no trailing period. Say in the pull request what changed and
|
|
46
|
+
how you checked it.
|
trainlint-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Satvik Jain
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
trainlint-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: trainlint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline checker and agent skills for ML bugs that do not crash: leakage, train/eval mode, loss and metric mistakes.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Satvik-jain/trainlint
|
|
6
|
+
Project-URL: Issues, https://github.com/Satvik-jain/trainlint/issues
|
|
7
|
+
Author: Satvik Jain
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agent-skills,data-leakage,linter,machine-learning,pytorch,scikit-learn,static-analysis
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# trainlint
|
|
23
|
+
|
|
24
|
+
Agent skills and an offline checker for ML bugs that do not crash.
|
|
25
|
+
|
|
26
|
+
[](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml)
|
|
27
|
+
[](LICENSE)
|
|
28
|
+
|
|
29
|
+

|
|
30
|
+
|
|
31
|
+
The 5,000 features and the labels are random, so no model can beat 50%. Picking features before
|
|
32
|
+
cross-validating reports 85%. The GIF is rendered from the real output of the commands shown;
|
|
33
|
+
`assets/make_assets.py` rebuilds it.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```console
|
|
38
|
+
uvx trainlint check .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
or `pipx install trainlint` / `pip install trainlint`. It needs Python 3.10 or newer, has no
|
|
42
|
+
dependencies, and never touches the network or a GPU.
|
|
43
|
+
|
|
44
|
+
## Quickstart
|
|
45
|
+
|
|
46
|
+
```console
|
|
47
|
+
uvx trainlint check train.py # find the bugs in a script, a folder, or a repo
|
|
48
|
+
uvx trainlint skills install # teach your coding agent to avoid them
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`skills install` copies four skills into `./.claude/skills`. Use `--agent agents` for
|
|
52
|
+
`./.agents/skills`, `--global` for your user folder, or `--dest DIR` for any other place.
|
|
53
|
+
|
|
54
|
+
More ways to run the checker:
|
|
55
|
+
|
|
56
|
+
```console
|
|
57
|
+
trainlint check src/ --ignore TL005 # skip a rule
|
|
58
|
+
trainlint check . --format json # for scripts and CI
|
|
59
|
+
trainlint rules # list every rule
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Exit code 0 means no findings, 1 means findings, 2 means a usage error. To silence a finding on
|
|
63
|
+
one line:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
scaler.fit(X_all) # trainlint: ignore[TL002] fitting on the pooled set is intended here
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
In GitHub Actions:
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
- run: uvx trainlint check .
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Why it exists
|
|
76
|
+
|
|
77
|
+
ML code fails quietly. A scaler fitted before the split, a model left in eval mode, or a softmax
|
|
78
|
+
in front of `CrossEntropyLoss` still trains and still prints a number. Coding agents and people
|
|
79
|
+
both write these mistakes, and general linters do not know what the code is for. trainlint is a
|
|
80
|
+
small offline checker for the common cases, plus skills that tell an agent how to avoid them in
|
|
81
|
+
the first place.
|
|
82
|
+
|
|
83
|
+
## What it does
|
|
84
|
+
|
|
85
|
+
- Twelve rules, TL001 to TL012, listed below and explained with examples in
|
|
86
|
+
[docs/rules.md](docs/rules.md).
|
|
87
|
+
- Each finding shows the line, a pointer into the code, and a one-line fix.
|
|
88
|
+
- Pure standard library. It parses the code and never runs it.
|
|
89
|
+
- Text or JSON output, rule selection, and per-line ignore comments.
|
|
90
|
+
- Four [Agent Skills](https://agentskills.io/specification) that cover the same ground and run
|
|
91
|
+
the checker when it is installed.
|
|
92
|
+
|
|
93
|
+
| Rule | Flags |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| TL001 | Scaler, encoder, selector or resampler fitted on all rows before a split or cross-validation |
|
|
96
|
+
| TL002 | Transform or model fitted on a validation or test split |
|
|
97
|
+
| TL003 | `model.eval()` inside a training loop with no `model.train()` to switch back |
|
|
98
|
+
| TL004 | Evaluation under `no_grad` in a script that never calls `model.eval()` |
|
|
99
|
+
| TL005 | Training script that never sets a random seed |
|
|
100
|
+
| TL006 | float16 autocast without a `GradScaler`, or a scaled loss stepped without `scaler.step` |
|
|
101
|
+
| TL007 | Script that resumes from checkpoints but saves no optimizer state |
|
|
102
|
+
| TL008 | `softmax` or `sigmoid` at the end of a model trained with a logits loss |
|
|
103
|
+
| TL009 | Gradients never cleared, or cleared between `backward()` and `step()` |
|
|
104
|
+
| TL010 | Loss added to a running total or list as a tensor, keeping every graph alive |
|
|
105
|
+
| TL011 | `y_true` and `y_pred` swapped, or class labels passed to `roc_auc_score` |
|
|
106
|
+
| TL012 | Validation or test `DataLoader` with `drop_last=True` |
|
|
107
|
+
|
|
108
|
+
| Skill | Covers |
|
|
109
|
+
| --- | --- |
|
|
110
|
+
| `ml-data-leakage` | Splits, pipelines, grouped and time-ordered data, resampling, the test set |
|
|
111
|
+
| `ml-training-loop` | Modes, gradient order, loss inputs, shapes, seeds, mixed precision |
|
|
112
|
+
| `ml-metrics-eval` | Argument order, scores versus labels, averaging, imbalance, variance |
|
|
113
|
+
| `ml-checkpoint-resume` | What a checkpoint must hold, best versus last, verifying a resume |
|
|
114
|
+
|
|
115
|
+
## How far to trust it
|
|
116
|
+
|
|
117
|
+
The checks parse each file with Python's `ast` module and match calls by name. There is no type
|
|
118
|
+
inference and no cross-file analysis, so:
|
|
119
|
+
|
|
120
|
+
- Splits are recognized by variable names such as `X_train`, `val_loader` and `y_test`.
|
|
121
|
+
- A bug whose cause sits in another file is not seen.
|
|
122
|
+
- A heuristic can be wrong. When it is, add an ignore comment and open an issue.
|
|
123
|
+
|
|
124
|
+
`python -m bench` runs the checker over a corpus of 18 buggy scripts, one or two per rule, each
|
|
125
|
+
with a fixed twin:
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
rule buggy files caught false alarms on fixed twins
|
|
129
|
+
TL001 3 3 0
|
|
130
|
+
TL002 1 1 0
|
|
131
|
+
TL003 1 1 0
|
|
132
|
+
TL004 1 1 0
|
|
133
|
+
TL005 1 1 0
|
|
134
|
+
TL006 2 2 0
|
|
135
|
+
TL007 1 1 0
|
|
136
|
+
TL008 2 2 0
|
|
137
|
+
TL009 2 2 0
|
|
138
|
+
TL010 1 1 0
|
|
139
|
+
TL011 2 2 0
|
|
140
|
+
TL012 1 1 0
|
|
141
|
+
total 18 18 0
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
I wrote that corpus together with the rules, so it guards against regressions and says nothing
|
|
145
|
+
about accuracy on code I have not seen.
|
|
146
|
+
|
|
147
|
+
For a check on someone else's code, I ran it over all 93 Python files of
|
|
148
|
+
[pytorch/examples](https://github.com/pytorch/examples) at commit `acc295d` and read every
|
|
149
|
+
finding:
|
|
150
|
+
|
|
151
|
+
```console
|
|
152
|
+
git clone https://github.com/pytorch/examples && cd examples
|
|
153
|
+
git checkout acc295dc7b90714f1bf47f06004fc19a7fe235c4
|
|
154
|
+
trainlint check .
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
It reports 21 findings. By my reading, 19 are real: 15 scripts never set a seed (one declares a
|
|
158
|
+
`--seed` option and never uses it), 3 tensor-parallel scripts never clear gradients, and one script
|
|
159
|
+
resumes from a snapshot without saving optimizer state. Two more are correct as written but
|
|
160
|
+
harmless, because those models have no dropout or batch norm to leave in training mode. The first
|
|
161
|
+
run also flagged three distributed-autograd scripts for missing `zero_grad()`, which was a false
|
|
162
|
+
positive that is now fixed. This is one repository of tutorial code and I did the reviewing, so
|
|
163
|
+
treat it as a spot check. It tells you nothing about what the rules miss.
|
|
164
|
+
|
|
165
|
+
## Compared with other tools
|
|
166
|
+
|
|
167
|
+
| Tool | What it covers | Where trainlint differs |
|
|
168
|
+
| --- | --- | --- |
|
|
169
|
+
| ruff, pylint, flake8 | General Python style and bugs | None of their rules know about splits, eval mode or metrics. Use them alongside trainlint. |
|
|
170
|
+
| [TorchFix](https://github.com/meta-pytorch/torchfix) | Removed and deprecated PyTorch functions, `requires_grad` typos, unsafe `torch.load` | Its README lists no rules for training logic. trainlint does not check API deprecations. |
|
|
171
|
+
| [mllint](https://github.com/bvobart/mllint) | Repository-level quality of ML projects: version control, dependencies, code quality | It looks at project hygiene. trainlint reads training and evaluation code. |
|
|
172
|
+
| [Deepchecks](https://github.com/deepchecks/deepchecks) | Checks that run on real datasets and models, including drift | Deepchecks needs your data and model loaded. trainlint reads source only and runs nothing. |
|
|
173
|
+
| General skill collections | Engineering process for coding agents | They are not specific to ML. trainlint's skills target training and evaluation code and come with a checker. |
|
|
174
|
+
|
|
175
|
+
## Roadmap
|
|
176
|
+
|
|
177
|
+
- Jupyter notebook support
|
|
178
|
+
- SARIF output for GitHub code scanning
|
|
179
|
+
- Settings in `pyproject.toml`
|
|
180
|
+
- More rules: training loaders without shuffling, `retain_graph` in a loop, normalization on a
|
|
181
|
+
whole DataFrame before the split
|
|
182
|
+
- A benchmark that measures how often an agent writes these bugs with and without the skills
|
|
183
|
+
(needs an API key, so it will be optional)
|
|
184
|
+
|
|
185
|
+
Issues labeled [good first issue](https://github.com/Satvik-jain/trainlint/labels/good%20first%20issue)
|
|
186
|
+
are small and have a clear definition of done.
|
|
187
|
+
|
|
188
|
+
## Contributing
|
|
189
|
+
|
|
190
|
+
False positives and missed bugs are the most useful reports. See
|
|
191
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) for setup and for how to add a rule.
|
|
192
|
+
|
|
193
|
+
```console
|
|
194
|
+
uv sync
|
|
195
|
+
uv run pytest
|
|
196
|
+
uv run ruff check . && uv run ruff format --check .
|
|
197
|
+
uv run python -m bench
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
MIT
|
|
203
|
+
|
|
204
|
+
If trainlint saved you an afternoon, a star on the repository helps other people find it.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# trainlint
|
|
2
|
+
|
|
3
|
+
Agent skills and an offline checker for ML bugs that do not crash.
|
|
4
|
+
|
|
5
|
+
[](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
The 5,000 features and the labels are random, so no model can beat 50%. Picking features before
|
|
11
|
+
cross-validating reports 85%. The GIF is rendered from the real output of the commands shown;
|
|
12
|
+
`assets/make_assets.py` rebuilds it.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```console
|
|
17
|
+
uvx trainlint check .
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
or `pipx install trainlint` / `pip install trainlint`. It needs Python 3.10 or newer, has no
|
|
21
|
+
dependencies, and never touches the network or a GPU.
|
|
22
|
+
|
|
23
|
+
## Quickstart
|
|
24
|
+
|
|
25
|
+
```console
|
|
26
|
+
uvx trainlint check train.py # find the bugs in a script, a folder, or a repo
|
|
27
|
+
uvx trainlint skills install # teach your coding agent to avoid them
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`skills install` copies four skills into `./.claude/skills`. Use `--agent agents` for
|
|
31
|
+
`./.agents/skills`, `--global` for your user folder, or `--dest DIR` for any other place.
|
|
32
|
+
|
|
33
|
+
More ways to run the checker:
|
|
34
|
+
|
|
35
|
+
```console
|
|
36
|
+
trainlint check src/ --ignore TL005 # skip a rule
|
|
37
|
+
trainlint check . --format json # for scripts and CI
|
|
38
|
+
trainlint rules # list every rule
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Exit code 0 means no findings, 1 means findings, 2 means a usage error. To silence a finding on
|
|
42
|
+
one line:
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
scaler.fit(X_all) # trainlint: ignore[TL002] fitting on the pooled set is intended here
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
In GitHub Actions:
|
|
49
|
+
|
|
50
|
+
```yaml
|
|
51
|
+
- run: uvx trainlint check .
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Why it exists
|
|
55
|
+
|
|
56
|
+
ML code fails quietly. A scaler fitted before the split, a model left in eval mode, or a softmax
|
|
57
|
+
in front of `CrossEntropyLoss` still trains and still prints a number. Coding agents and people
|
|
58
|
+
both write these mistakes, and general linters do not know what the code is for. trainlint is a
|
|
59
|
+
small offline checker for the common cases, plus skills that tell an agent how to avoid them in
|
|
60
|
+
the first place.
|
|
61
|
+
|
|
62
|
+
## What it does
|
|
63
|
+
|
|
64
|
+
- Twelve rules, TL001 to TL012, listed below and explained with examples in
|
|
65
|
+
[docs/rules.md](docs/rules.md).
|
|
66
|
+
- Each finding shows the line, a pointer into the code, and a one-line fix.
|
|
67
|
+
- Pure standard library. It parses the code and never runs it.
|
|
68
|
+
- Text or JSON output, rule selection, and per-line ignore comments.
|
|
69
|
+
- Four [Agent Skills](https://agentskills.io/specification) that cover the same ground and run
|
|
70
|
+
the checker when it is installed.
|
|
71
|
+
|
|
72
|
+
| Rule | Flags |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| TL001 | Scaler, encoder, selector or resampler fitted on all rows before a split or cross-validation |
|
|
75
|
+
| TL002 | Transform or model fitted on a validation or test split |
|
|
76
|
+
| TL003 | `model.eval()` inside a training loop with no `model.train()` to switch back |
|
|
77
|
+
| TL004 | Evaluation under `no_grad` in a script that never calls `model.eval()` |
|
|
78
|
+
| TL005 | Training script that never sets a random seed |
|
|
79
|
+
| TL006 | float16 autocast without a `GradScaler`, or a scaled loss stepped without `scaler.step` |
|
|
80
|
+
| TL007 | Script that resumes from checkpoints but saves no optimizer state |
|
|
81
|
+
| TL008 | `softmax` or `sigmoid` at the end of a model trained with a logits loss |
|
|
82
|
+
| TL009 | Gradients never cleared, or cleared between `backward()` and `step()` |
|
|
83
|
+
| TL010 | Loss added to a running total or list as a tensor, keeping every graph alive |
|
|
84
|
+
| TL011 | `y_true` and `y_pred` swapped, or class labels passed to `roc_auc_score` |
|
|
85
|
+
| TL012 | Validation or test `DataLoader` with `drop_last=True` |
|
|
86
|
+
|
|
87
|
+
| Skill | Covers |
|
|
88
|
+
| --- | --- |
|
|
89
|
+
| `ml-data-leakage` | Splits, pipelines, grouped and time-ordered data, resampling, the test set |
|
|
90
|
+
| `ml-training-loop` | Modes, gradient order, loss inputs, shapes, seeds, mixed precision |
|
|
91
|
+
| `ml-metrics-eval` | Argument order, scores versus labels, averaging, imbalance, variance |
|
|
92
|
+
| `ml-checkpoint-resume` | What a checkpoint must hold, best versus last, verifying a resume |
|
|
93
|
+
|
|
94
|
+
## How far to trust it
|
|
95
|
+
|
|
96
|
+
The checks parse each file with Python's `ast` module and match calls by name. There is no type
|
|
97
|
+
inference and no cross-file analysis, so:
|
|
98
|
+
|
|
99
|
+
- Splits are recognized by variable names such as `X_train`, `val_loader` and `y_test`.
|
|
100
|
+
- A bug whose cause sits in another file is not seen.
|
|
101
|
+
- A heuristic can be wrong. When it is, add an ignore comment and open an issue.
|
|
102
|
+
|
|
103
|
+
`python -m bench` runs the checker over a corpus of 18 buggy scripts, one or two per rule, each
|
|
104
|
+
with a fixed twin:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
rule buggy files caught false alarms on fixed twins
|
|
108
|
+
TL001 3 3 0
|
|
109
|
+
TL002 1 1 0
|
|
110
|
+
TL003 1 1 0
|
|
111
|
+
TL004 1 1 0
|
|
112
|
+
TL005 1 1 0
|
|
113
|
+
TL006 2 2 0
|
|
114
|
+
TL007 1 1 0
|
|
115
|
+
TL008 2 2 0
|
|
116
|
+
TL009 2 2 0
|
|
117
|
+
TL010 1 1 0
|
|
118
|
+
TL011 2 2 0
|
|
119
|
+
TL012 1 1 0
|
|
120
|
+
total 18 18 0
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
I wrote that corpus together with the rules, so it guards against regressions and says nothing
|
|
124
|
+
about accuracy on code I have not seen.
|
|
125
|
+
|
|
126
|
+
For a check on someone else's code, I ran it over all 93 Python files of
|
|
127
|
+
[pytorch/examples](https://github.com/pytorch/examples) at commit `acc295d` and read every
|
|
128
|
+
finding:
|
|
129
|
+
|
|
130
|
+
```console
|
|
131
|
+
git clone https://github.com/pytorch/examples && cd examples
|
|
132
|
+
git checkout acc295dc7b90714f1bf47f06004fc19a7fe235c4
|
|
133
|
+
trainlint check .
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
It reports 21 findings. By my reading, 19 are real: 15 scripts never set a seed (one declares a
|
|
137
|
+
`--seed` option and never uses it), 3 tensor-parallel scripts never clear gradients, and one script
|
|
138
|
+
resumes from a snapshot without saving optimizer state. Two more are correct as written but
|
|
139
|
+
harmless, because those models have no dropout or batch norm to leave in training mode. The first
|
|
140
|
+
run also flagged three distributed-autograd scripts for missing `zero_grad()`, which was a false
|
|
141
|
+
positive that is now fixed. This is one repository of tutorial code and I did the reviewing, so
|
|
142
|
+
treat it as a spot check. It tells you nothing about what the rules miss.
|
|
143
|
+
|
|
144
|
+
## Compared with other tools
|
|
145
|
+
|
|
146
|
+
| Tool | What it covers | Where trainlint differs |
|
|
147
|
+
| --- | --- | --- |
|
|
148
|
+
| ruff, pylint, flake8 | General Python style and bugs | None of their rules know about splits, eval mode or metrics. Use them alongside trainlint. |
|
|
149
|
+
| [TorchFix](https://github.com/meta-pytorch/torchfix) | Removed and deprecated PyTorch functions, `requires_grad` typos, unsafe `torch.load` | Its README lists no rules for training logic. trainlint does not check API deprecations. |
|
|
150
|
+
| [mllint](https://github.com/bvobart/mllint) | Repository-level quality of ML projects: version control, dependencies, code quality | It looks at project hygiene. trainlint reads training and evaluation code. |
|
|
151
|
+
| [Deepchecks](https://github.com/deepchecks/deepchecks) | Checks that run on real datasets and models, including drift | Deepchecks needs your data and model loaded. trainlint reads source only and runs nothing. |
|
|
152
|
+
| General skill collections | Engineering process for coding agents | They are not specific to ML. trainlint's skills target training and evaluation code and come with a checker. |
|
|
153
|
+
|
|
154
|
+
## Roadmap
|
|
155
|
+
|
|
156
|
+
- Jupyter notebook support
|
|
157
|
+
- SARIF output for GitHub code scanning
|
|
158
|
+
- Settings in `pyproject.toml`
|
|
159
|
+
- More rules: training loaders without shuffling, `retain_graph` in a loop, normalization on a
|
|
160
|
+
whole DataFrame before the split
|
|
161
|
+
- A benchmark that measures how often an agent writes these bugs with and without the skills
|
|
162
|
+
(needs an API key, so it will be optional)
|
|
163
|
+
|
|
164
|
+
Issues labeled [good first issue](https://github.com/Satvik-jain/trainlint/labels/good%20first%20issue)
|
|
165
|
+
are small and have a clear definition of done.
|
|
166
|
+
|
|
167
|
+
## Contributing
|
|
168
|
+
|
|
169
|
+
False positives and missed bugs are the most useful reports. See
|
|
170
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) for setup and for how to add a rule.
|
|
171
|
+
|
|
172
|
+
```console
|
|
173
|
+
uv sync
|
|
174
|
+
uv run pytest
|
|
175
|
+
uv run ruff check . && uv run ruff format --check .
|
|
176
|
+
uv run python -m bench
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## License
|
|
180
|
+
|
|
181
|
+
MIT
|
|
182
|
+
|
|
183
|
+
If trainlint saved you an afternoon, a star on the repository helps other people find it.
|
|
Binary file
|