trainlint 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. trainlint-0.1.0/.claude/settings.local.json +7 -0
  2. trainlint-0.1.0/.gitattributes +1 -0
  3. trainlint-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +23 -0
  4. trainlint-0.1.0/.github/ISSUE_TEMPLATE/config.yml +1 -0
  5. trainlint-0.1.0/.github/ISSUE_TEMPLATE/false_positive.yml +30 -0
  6. trainlint-0.1.0/.github/ISSUE_TEMPLATE/rule_idea.yml +27 -0
  7. trainlint-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +8 -0
  8. trainlint-0.1.0/.github/workflows/ci.yml +43 -0
  9. trainlint-0.1.0/.gitignore +8 -0
  10. trainlint-0.1.0/CHANGELOG.md +14 -0
  11. trainlint-0.1.0/CONTRIBUTING.md +46 -0
  12. trainlint-0.1.0/LICENSE +21 -0
  13. trainlint-0.1.0/PKG-INFO +204 -0
  14. trainlint-0.1.0/README.md +183 -0
  15. trainlint-0.1.0/assets/demo.gif +0 -0
  16. trainlint-0.1.0/assets/make_assets.py +187 -0
  17. trainlint-0.1.0/assets/social-preview.png +0 -0
  18. trainlint-0.1.0/bench/__init__.py +0 -0
  19. trainlint-0.1.0/bench/__main__.py +25 -0
  20. trainlint-0.1.0/bench/buggy/tl001_scaler_before_split.py +11 -0
  21. trainlint-0.1.0/bench/buggy/tl001_select_k_best_cv.py +11 -0
  22. trainlint-0.1.0/bench/buggy/tl001_smote_before_split.py +11 -0
  23. trainlint-0.1.0/bench/buggy/tl002_scaler_fit_on_test.py +13 -0
  24. trainlint-0.1.0/bench/buggy/tl003_eval_left_on.py +28 -0
  25. trainlint-0.1.0/bench/buggy/tl004_no_grad_without_eval.py +31 -0
  26. trainlint-0.1.0/bench/buggy/tl005_no_seed.py +31 -0
  27. trainlint-0.1.0/bench/buggy/tl006_fp16_without_scaler.py +33 -0
  28. trainlint-0.1.0/bench/buggy/tl006_scaled_loss_plain_step.py +35 -0
  29. trainlint-0.1.0/bench/buggy/tl007_resume_without_optimizer_state.py +33 -0
  30. trainlint-0.1.0/bench/buggy/tl008_sigmoid_before_bce_with_logits.py +17 -0
  31. trainlint-0.1.0/bench/buggy/tl008_softmax_before_cross_entropy.py +41 -0
  32. trainlint-0.1.0/bench/buggy/tl009_gradients_never_cleared.py +31 -0
  33. trainlint-0.1.0/bench/buggy/tl009_zero_grad_after_backward.py +32 -0
  34. trainlint-0.1.0/bench/buggy/tl010_running_loss_tensor.py +34 -0
  35. trainlint-0.1.0/bench/buggy/tl011_auc_from_class_labels.py +11 -0
  36. trainlint-0.1.0/bench/buggy/tl011_swapped_metric_arguments.py +11 -0
  37. trainlint-0.1.0/bench/buggy/tl012_validation_loader_drops_last.py +32 -0
  38. trainlint-0.1.0/bench/corpus.py +64 -0
  39. trainlint-0.1.0/bench/fixed/tl001_scaler_before_split.py +10 -0
  40. trainlint-0.1.0/bench/fixed/tl001_select_k_best_cv.py +10 -0
  41. trainlint-0.1.0/bench/fixed/tl001_smote_before_split.py +10 -0
  42. trainlint-0.1.0/bench/fixed/tl002_scaler_fit_on_test.py +12 -0
  43. trainlint-0.1.0/bench/fixed/tl003_eval_left_on.py +27 -0
  44. trainlint-0.1.0/bench/fixed/tl004_no_grad_without_eval.py +31 -0
  45. trainlint-0.1.0/bench/fixed/tl005_no_seed.py +31 -0
  46. trainlint-0.1.0/bench/fixed/tl006_fp16_without_scaler.py +34 -0
  47. trainlint-0.1.0/bench/fixed/tl006_scaled_loss_plain_step.py +34 -0
  48. trainlint-0.1.0/bench/fixed/tl007_resume_without_optimizer_state.py +39 -0
  49. trainlint-0.1.0/bench/fixed/tl008_sigmoid_before_bce_with_logits.py +16 -0
  50. trainlint-0.1.0/bench/fixed/tl008_softmax_before_cross_entropy.py +40 -0
  51. trainlint-0.1.0/bench/fixed/tl009_gradients_never_cleared.py +31 -0
  52. trainlint-0.1.0/bench/fixed/tl009_zero_grad_after_backward.py +31 -0
  53. trainlint-0.1.0/bench/fixed/tl010_running_loss_tensor.py +33 -0
  54. trainlint-0.1.0/bench/fixed/tl011_auc_from_class_labels.py +10 -0
  55. trainlint-0.1.0/bench/fixed/tl011_swapped_metric_arguments.py +10 -0
  56. trainlint-0.1.0/bench/fixed/tl012_validation_loader_drops_last.py +31 -0
  57. trainlint-0.1.0/docs/rules.md +250 -0
  58. trainlint-0.1.0/examples/noise_leak.py +21 -0
  59. trainlint-0.1.0/examples/noise_leak_fixed.py +18 -0
  60. trainlint-0.1.0/notes/publish.sh +113 -0
  61. trainlint-0.1.0/notes/release-notes.md +15 -0
  62. trainlint-0.1.0/notes/spec.md +50 -0
  63. trainlint-0.1.0/pyproject.toml +53 -0
  64. trainlint-0.1.0/src/trainlint/__init__.py +3 -0
  65. trainlint-0.1.0/src/trainlint/__main__.py +3 -0
  66. trainlint-0.1.0/src/trainlint/astutil.py +122 -0
  67. trainlint-0.1.0/src/trainlint/cli.py +118 -0
  68. trainlint-0.1.0/src/trainlint/engine.py +132 -0
  69. trainlint-0.1.0/src/trainlint/report.py +72 -0
  70. trainlint-0.1.0/src/trainlint/rules/__init__.py +19 -0
  71. trainlint-0.1.0/src/trainlint/rules/base.py +21 -0
  72. trainlint-0.1.0/src/trainlint/rules/checkpoints.py +68 -0
  73. trainlint-0.1.0/src/trainlint/rules/evaluation.py +117 -0
  74. trainlint-0.1.0/src/trainlint/rules/leakage.py +115 -0
  75. trainlint-0.1.0/src/trainlint/rules/mixed_precision.py +85 -0
  76. trainlint-0.1.0/src/trainlint/rules/training.py +302 -0
  77. trainlint-0.1.0/src/trainlint/skills/ml-checkpoint-resume/SKILL.md +70 -0
  78. trainlint-0.1.0/src/trainlint/skills/ml-data-leakage/SKILL.md +73 -0
  79. trainlint-0.1.0/src/trainlint/skills/ml-metrics-eval/SKILL.md +53 -0
  80. trainlint-0.1.0/src/trainlint/skills/ml-training-loop/SKILL.md +98 -0
  81. trainlint-0.1.0/src/trainlint/skills_install.py +36 -0
  82. trainlint-0.1.0/tests/conftest.py +17 -0
  83. trainlint-0.1.0/tests/test_bench.py +27 -0
  84. trainlint-0.1.0/tests/test_cli.py +72 -0
  85. trainlint-0.1.0/tests/test_docs.py +38 -0
  86. trainlint-0.1.0/tests/test_engine.py +64 -0
  87. trainlint-0.1.0/tests/test_leakage.py +135 -0
  88. trainlint-0.1.0/tests/test_precision_checkpoints_metrics.py +188 -0
  89. trainlint-0.1.0/tests/test_skills.py +65 -0
  90. trainlint-0.1.0/tests/test_training.py +322 -0
  91. trainlint-0.1.0/uv.lock +202 -0
@@ -0,0 +1,7 @@
1
+ {
2
+ "attribution": {
3
+ "commit": "",
4
+ "pr": "",
5
+ "sessionUrl": false
6
+ }
7
+ }
@@ -0,0 +1 @@
1
+ * text=auto eol=lf
@@ -0,0 +1,23 @@
1
+ name: Bug report
2
+ description: Something other than a wrong finding is broken
3
+ labels: ["bug"]
4
+ body:
5
+ - type: textarea
6
+ id: what
7
+ attributes:
8
+ label: What happened
9
+ description: The command you ran, what you expected, and what you got.
10
+ validations:
11
+ required: true
12
+ - type: input
13
+ id: version
14
+ attributes:
15
+ label: Output of `trainlint --version`
16
+ validations:
17
+ required: true
18
+ - type: input
19
+ id: platform
20
+ attributes:
21
+ label: Operating system and Python version
22
+ validations:
23
+ required: true
@@ -0,0 +1 @@
1
+ blank_issues_enabled: false
@@ -0,0 +1,30 @@
1
+ name: False positive
2
+ description: trainlint flagged code that is correct
3
+ labels: ["false positive"]
4
+ body:
5
+ - type: input
6
+ id: rule
7
+ attributes:
8
+ label: Rule
9
+ placeholder: TL001
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: snippet
14
+ attributes:
15
+ label: Smallest snippet that is flagged
16
+ render: python
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: why
21
+ attributes:
22
+ label: Why this code is correct
23
+ validations:
24
+ required: true
25
+ - type: input
26
+ id: version
27
+ attributes:
28
+ label: Output of `trainlint --version`
29
+ validations:
30
+ required: true
@@ -0,0 +1,27 @@
1
+ name: Rule idea
2
+ description: A bug that trainlint should catch
3
+ labels: ["new rule"]
4
+ body:
5
+ - type: textarea
6
+ id: bug
7
+ attributes:
8
+ label: The bug
9
+ description: What goes wrong, and how you would notice it (or not).
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: example
14
+ attributes:
15
+ label: Minimal example
16
+ render: python
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: signal
21
+ attributes:
22
+ label: What in the code gives it away
23
+ description: |
24
+ Rules work on one file's syntax tree, with no type information. Name the calls, names or
25
+ ordering a check could look at, and a sensible script that looks similar but is fine.
26
+ validations:
27
+ required: true
@@ -0,0 +1,8 @@
1
+ ## What changed
2
+
3
+ ## How I checked it
4
+
5
+ - [ ] `uv run pytest` passes
6
+ - [ ] `uv run ruff check . && uv run ruff format --check .` passes
7
+ - [ ] New or changed rule: buggy and fixed files added under `bench/`, example added to `docs/rules.md`
8
+ - [ ] New or changed rule: I read what it reports on at least one real codebase
@@ -0,0 +1,43 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ name: test (${{ matrix.os }}, python ${{ matrix.python }})
11
+ runs-on: ${{ matrix.os }}
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ os: [ubuntu-latest, windows-latest, macos-latest]
16
+ python: ["3.10", "3.13"]
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: astral-sh/setup-uv@v5
20
+ - run: uv sync --locked --python ${{ matrix.python }}
21
+ - run: uv run pytest -q
22
+ - run: uv run python -m bench
23
+
24
+ lint:
25
+ runs-on: ubuntu-latest
26
+ steps:
27
+ - uses: actions/checkout@v4
28
+ - uses: astral-sh/setup-uv@v5
29
+ - run: uv sync --locked
30
+ - run: uv run ruff check .
31
+ - run: uv run ruff format --check .
32
+
33
+ package:
34
+ runs-on: ubuntu-latest
35
+ steps:
36
+ - uses: actions/checkout@v4
37
+ - uses: astral-sh/setup-uv@v5
38
+ - run: uv build
39
+ - name: Install the wheel into a clean environment and run it
40
+ run: |
41
+ uvx --from dist/*.whl trainlint skills install --dest "$RUNNER_TEMP/skills"
42
+ test -f "$RUNNER_TEMP/skills/ml-training-loop/SKILL.md"
43
+ uvx --from dist/*.whl trainlint check --exit-zero bench/buggy
@@ -0,0 +1,8 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ dist/
7
+ build/
8
+ *.egg-info/
@@ -0,0 +1,14 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 - 2026-11-03
4
+
5
+ First release.
6
+
7
+ - `trainlint check` with 12 rules (TL001 to TL012) for data leakage, train and eval mode,
8
+ seeding, mixed precision, checkpoints, loss inputs, gradient clearing, tensor accumulation,
9
+ metric misuse and evaluation loaders. Text and JSON output, `--select`, `--ignore`,
10
+ `--exit-zero`, `--color`, and `# trainlint: ignore[RULE]` comments.
11
+ - Four Agent Skills (`ml-data-leakage`, `ml-training-loop`, `ml-metrics-eval`,
12
+ `ml-checkpoint-resume`) and `trainlint skills install`.
13
+ - Regression corpus of 18 buggy scripts with fixed twins (`python -m bench`).
14
+ - Per-rule reference in `docs/rules.md`.
@@ -0,0 +1,46 @@
1
+ # Contributing
2
+
3
+ Reports of false positives and missed bugs are the most useful thing you can send. A rule that
4
+ cries wolf gets switched off, so precision matters more here than coverage.
5
+
6
+ ## Set up
7
+
8
+ You need [uv](https://docs.astral.sh/uv/) and Python 3.10 or newer.
9
+
10
+ ```console
11
+ git clone https://github.com/Satvik-jain/trainlint
12
+ cd trainlint
13
+ uv sync
14
+ uv run pytest
15
+ uv run ruff check . && uv run ruff format --check .
16
+ uv run python -m bench
17
+ ```
18
+
19
+ ## Reporting a problem
20
+
21
+ - **False positive:** use the "False positive" issue form. Paste the smallest snippet that is
22
+ flagged and say why it is fine.
23
+ - **Missed bug:** use the "Rule idea" form. Show the bug and say what in the code gives it away.
24
+ A rule has to work from one file's syntax tree, with no types and no imports resolved.
25
+
26
+ ## Adding or changing a rule
27
+
28
+ 1. Write the check as a function `(tree: ast.Module) -> Iterator[Hit]` in the module for its
29
+ topic under `src/trainlint/rules/`, and register it in that module's `RULES`. Helpers for
30
+ matching calls and names live in `src/trainlint/astutil.py`.
31
+ 2. Add unit tests with both flagged and clean snippets. Clean snippets matter more: write down
32
+ the ways a sensible script could look like the bug without being it.
33
+ 3. Add a buggy script and its fixed twin to `bench/buggy/` and `bench/fixed/`. The buggy file
34
+ starts with `# expect: TLxxx`.
35
+ 4. Add the rule to `docs/rules.md` with one flagged and one clean example. A test runs both.
36
+ 5. If a bundled skill should mention it, add it to the skill under `src/trainlint/skills/`.
37
+ A test checks that skills only cite rules that exist and that every rule is cited.
38
+
39
+ A new rule should be quiet on ordinary code. If you can, run it over a few real repositories and
40
+ read what it reports before opening the pull request.
41
+
42
+ ## Commits and pull requests
43
+
44
+ Keep commits small and use Conventional Commit prefixes (`feat`, `fix`, `docs`, `test`, `chore`,
45
+ `refactor`, `ci`), imperative mood, no trailing period. Say in the pull request what changed and
46
+ how you checked it.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Satvik Jain
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,204 @@
1
+ Metadata-Version: 2.5
2
+ Name: trainlint
3
+ Version: 0.1.0
4
+ Summary: Offline checker and agent skills for ML bugs that do not crash: leakage, train/eval mode, loss and metric mistakes.
5
+ Project-URL: Homepage, https://github.com/Satvik-jain/trainlint
6
+ Project-URL: Issues, https://github.com/Satvik-jain/trainlint/issues
7
+ Author: Satvik Jain
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: agent-skills,data-leakage,linter,machine-learning,pytorch,scikit-learn,static-analysis
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Classifier: Topic :: Software Development :: Quality Assurance
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+
22
+ # trainlint
23
+
24
+ Agent skills and an offline checker for ML bugs that do not crash.
25
+
26
+ [![ci](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml/badge.svg)](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml)
27
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
28
+
29
+ ![Cross-validated accuracy on random labels: 85% with a leak, trainlint pointing at it, 48% after the fix](assets/demo.gif)
30
+
31
+ The 5,000 features and the labels are random, so no model can beat 50%. Picking features before
32
+ cross-validating reports 85%. The GIF is rendered from the real output of the commands shown;
33
+ `assets/make_assets.py` rebuilds it.
34
+
35
+ ## Install
36
+
37
+ ```console
38
+ uvx trainlint check .
39
+ ```
40
+
41
+ or `pipx install trainlint` / `pip install trainlint`. It needs Python 3.10 or newer, has no
42
+ dependencies, and never touches the network or a GPU.
43
+
44
+ ## Quickstart
45
+
46
+ ```console
47
+ uvx trainlint check train.py # find the bugs in a script, a folder, or a repo
48
+ uvx trainlint skills install # teach your coding agent to avoid them
49
+ ```
50
+
51
+ `skills install` copies four skills into `./.claude/skills`. Use `--agent agents` for
52
+ `./.agents/skills`, `--global` for your user folder, or `--dest DIR` for any other place.
53
+
54
+ More ways to run the checker:
55
+
56
+ ```console
57
+ trainlint check src/ --ignore TL005 # skip a rule
58
+ trainlint check . --format json # for scripts and CI
59
+ trainlint rules # list every rule
60
+ ```
61
+
62
+ Exit code 0 means no findings, 1 means findings, 2 means a usage error. To silence a finding on
63
+ one line:
64
+
65
+ ```python
66
+ scaler.fit(X_all) # trainlint: ignore[TL002] fitting on the pooled set is intended here
67
+ ```
68
+
69
+ In GitHub Actions:
70
+
71
+ ```yaml
72
+ - run: uvx trainlint check .
73
+ ```
74
+
75
+ ## Why it exists
76
+
77
+ ML code fails quietly. A scaler fitted before the split, a model left in eval mode, or a softmax
78
+ in front of `CrossEntropyLoss` still trains and still prints a number. Coding agents and people
79
+ both write these mistakes, and general linters do not know what the code is for. trainlint is a
80
+ small offline checker for the common cases, plus skills that tell an agent how to avoid them in
81
+ the first place.
82
+
83
+ ## What it does
84
+
85
+ - Twelve rules, TL001 to TL012, listed below and explained with examples in
86
+ [docs/rules.md](docs/rules.md).
87
+ - Each finding shows the line, a pointer into the code, and a one-line fix.
88
+ - Pure standard library. It parses the code and never runs it.
89
+ - Text or JSON output, rule selection, and per-line ignore comments.
90
+ - Four [Agent Skills](https://agentskills.io/specification) that cover the same ground and run
91
+ the checker when it is installed.
92
+
93
+ | Rule | Flags |
94
+ | --- | --- |
95
+ | TL001 | Scaler, encoder, selector or resampler fitted on all rows before a split or cross-validation |
96
+ | TL002 | Transform or model fitted on a validation or test split |
97
+ | TL003 | `model.eval()` inside a training loop with no `model.train()` to switch back |
98
+ | TL004 | Evaluation under `no_grad` in a script that never calls `model.eval()` |
99
+ | TL005 | Training script that never sets a random seed |
100
+ | TL006 | float16 autocast without a `GradScaler`, or a scaled loss stepped without `scaler.step` |
101
+ | TL007 | Script that resumes from checkpoints but saves no optimizer state |
102
+ | TL008 | `softmax` or `sigmoid` at the end of a model trained with a logits loss |
103
+ | TL009 | Gradients never cleared, or cleared between `backward()` and `step()` |
104
+ | TL010 | Loss added to a running total or list as a tensor, keeping every graph alive |
105
+ | TL011 | `y_true` and `y_pred` swapped, or class labels passed to `roc_auc_score` |
106
+ | TL012 | Validation or test `DataLoader` with `drop_last=True` |
107
+
108
+ | Skill | Covers |
109
+ | --- | --- |
110
+ | `ml-data-leakage` | Splits, pipelines, grouped and time-ordered data, resampling, the test set |
111
+ | `ml-training-loop` | Modes, gradient order, loss inputs, shapes, seeds, mixed precision |
112
+ | `ml-metrics-eval` | Argument order, scores versus labels, averaging, imbalance, variance |
113
+ | `ml-checkpoint-resume` | What a checkpoint must hold, best versus last, verifying a resume |
114
+
115
+ ## How far to trust it
116
+
117
+ The checks parse each file with Python's `ast` module and match calls by name. There is no type
118
+ inference and no cross-file analysis, so:
119
+
120
+ - Splits are recognized by variable names such as `X_train`, `val_loader` and `y_test`.
121
+ - A bug whose cause sits in another file is not seen.
122
+ - A heuristic can be wrong. When it is, add an ignore comment and open an issue.
123
+
124
+ `python -m bench` runs the checker over a corpus of 18 buggy scripts, one or two per rule, each
125
+ with a fixed twin:
126
+
127
+ ```
128
+ rule buggy files caught false alarms on fixed twins
129
+ TL001 3 3 0
130
+ TL002 1 1 0
131
+ TL003 1 1 0
132
+ TL004 1 1 0
133
+ TL005 1 1 0
134
+ TL006 2 2 0
135
+ TL007 1 1 0
136
+ TL008 2 2 0
137
+ TL009 2 2 0
138
+ TL010 1 1 0
139
+ TL011 2 2 0
140
+ TL012 1 1 0
141
+ total 18 18 0
142
+ ```
143
+
144
+ I wrote that corpus together with the rules, so it guards against regressions and says nothing
145
+ about accuracy on code I have not seen.
146
+
147
+ For a check on someone else's code, I ran it over all 93 Python files of
148
+ [pytorch/examples](https://github.com/pytorch/examples) at commit `acc295d` and read every
149
+ finding:
150
+
151
+ ```console
152
+ git clone https://github.com/pytorch/examples && cd examples
153
+ git checkout acc295dc7b90714f1bf47f06004fc19a7fe235c4
154
+ trainlint check .
155
+ ```
156
+
157
+ It reports 21 findings. By my reading, 19 are real: 15 scripts never set a seed (one declares a
158
+ `--seed` option and never uses it), 3 tensor-parallel scripts never clear gradients, and one script
159
+ resumes from a snapshot without saving optimizer state. Two more are correct as written but
160
+ harmless, because those models have no dropout or batch norm to leave in training mode. The first
161
+ run also flagged three distributed-autograd scripts for missing `zero_grad()`, which was a false
162
+ positive that is now fixed. This is one repository of tutorial code and I did the reviewing, so
163
+ treat it as a spot check. It tells you nothing about what the rules miss.
164
+
165
+ ## Compared with other tools
166
+
167
+ | Tool | What it covers | Where trainlint differs |
168
+ | --- | --- | --- |
169
+ | ruff, pylint, flake8 | General Python style and bugs | None of their rules know about splits, eval mode or metrics. Use them alongside trainlint. |
170
+ | [TorchFix](https://github.com/meta-pytorch/torchfix) | Removed and deprecated PyTorch functions, `requires_grad` typos, unsafe `torch.load` | Its README lists no rules for training logic. trainlint does not check API deprecations. |
171
+ | [mllint](https://github.com/bvobart/mllint) | Repository-level quality of ML projects: version control, dependencies, code quality | It looks at project hygiene. trainlint reads training and evaluation code. |
172
+ | [Deepchecks](https://github.com/deepchecks/deepchecks) | Checks that run on real datasets and models, including drift | Deepchecks needs your data and model loaded. trainlint reads source only and runs nothing. |
173
+ | General skill collections | Engineering process for coding agents | They are not specific to ML. trainlint's skills target training and evaluation code and come with a checker. |
174
+
175
+ ## Roadmap
176
+
177
+ - Jupyter notebook support
178
+ - SARIF output for GitHub code scanning
179
+ - Settings in `pyproject.toml`
180
+ - More rules: training loaders without shuffling, `retain_graph` in a loop, normalization on a
181
+ whole DataFrame before the split
182
+ - A benchmark that measures how often an agent writes these bugs with and without the skills
183
+ (needs an API key, so it will be optional)
184
+
185
+ Issues labeled [good first issue](https://github.com/Satvik-jain/trainlint/labels/good%20first%20issue)
186
+ are small and have a clear definition of done.
187
+
188
+ ## Contributing
189
+
190
+ False positives and missed bugs are the most useful reports. See
191
+ [CONTRIBUTING.md](CONTRIBUTING.md) for setup and for how to add a rule.
192
+
193
+ ```console
194
+ uv sync
195
+ uv run pytest
196
+ uv run ruff check . && uv run ruff format --check .
197
+ uv run python -m bench
198
+ ```
199
+
200
+ ## License
201
+
202
+ MIT
203
+
204
+ If trainlint saved you an afternoon, a star on the repository helps other people find it.
@@ -0,0 +1,183 @@
1
+ # trainlint
2
+
3
+ Agent skills and an offline checker for ML bugs that do not crash.
4
+
5
+ [![ci](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml/badge.svg)](https://github.com/Satvik-jain/trainlint/actions/workflows/ci.yml)
6
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
7
+
8
+ ![Cross-validated accuracy on random labels: 85% with a leak, trainlint pointing at it, 48% after the fix](assets/demo.gif)
9
+
10
+ The 5,000 features and the labels are random, so no model can beat 50%. Picking features before
11
+ cross-validating reports 85%. The GIF is rendered from the real output of the commands shown;
12
+ `assets/make_assets.py` rebuilds it.
13
+
14
+ ## Install
15
+
16
+ ```console
17
+ uvx trainlint check .
18
+ ```
19
+
20
+ or `pipx install trainlint` / `pip install trainlint`. It needs Python 3.10 or newer, has no
21
+ dependencies, and never touches the network or a GPU.
22
+
23
+ ## Quickstart
24
+
25
+ ```console
26
+ uvx trainlint check train.py # find the bugs in a script, a folder, or a repo
27
+ uvx trainlint skills install # teach your coding agent to avoid them
28
+ ```
29
+
30
+ `skills install` copies four skills into `./.claude/skills`. Use `--agent agents` for
31
+ `./.agents/skills`, `--global` for your user folder, or `--dest DIR` for any other place.
32
+
33
+ More ways to run the checker:
34
+
35
+ ```console
36
+ trainlint check src/ --ignore TL005 # skip a rule
37
+ trainlint check . --format json # for scripts and CI
38
+ trainlint rules # list every rule
39
+ ```
40
+
41
+ Exit code 0 means no findings, 1 means findings, 2 means a usage error. To silence a finding on
42
+ one line:
43
+
44
+ ```python
45
+ scaler.fit(X_all) # trainlint: ignore[TL002] fitting on the pooled set is intended here
46
+ ```
47
+
48
+ In GitHub Actions:
49
+
50
+ ```yaml
51
+ - run: uvx trainlint check .
52
+ ```
53
+
54
+ ## Why it exists
55
+
56
+ ML code fails quietly. A scaler fitted before the split, a model left in eval mode, or a softmax
57
+ in front of `CrossEntropyLoss` still trains and still prints a number. Coding agents and people
58
+ both write these mistakes, and general linters do not know what the code is for. trainlint is a
59
+ small offline checker for the common cases, plus skills that tell an agent how to avoid them in
60
+ the first place.
61
+
62
+ ## What it does
63
+
64
+ - Twelve rules, TL001 to TL012, listed below and explained with examples in
65
+ [docs/rules.md](docs/rules.md).
66
+ - Each finding shows the line, a pointer into the code, and a one-line fix.
67
+ - Pure standard library. It parses the code and never runs it.
68
+ - Text or JSON output, rule selection, and per-line ignore comments.
69
+ - Four [Agent Skills](https://agentskills.io/specification) that cover the same ground and run
70
+ the checker when it is installed.
71
+
72
+ | Rule | Flags |
73
+ | --- | --- |
74
+ | TL001 | Scaler, encoder, selector or resampler fitted on all rows before a split or cross-validation |
75
+ | TL002 | Transform or model fitted on a validation or test split |
76
+ | TL003 | `model.eval()` inside a training loop with no `model.train()` to switch back |
77
+ | TL004 | Evaluation under `no_grad` in a script that never calls `model.eval()` |
78
+ | TL005 | Training script that never sets a random seed |
79
+ | TL006 | float16 autocast without a `GradScaler`, or a scaled loss stepped without `scaler.step` |
80
+ | TL007 | Script that resumes from checkpoints but saves no optimizer state |
81
+ | TL008 | `softmax` or `sigmoid` at the end of a model trained with a logits loss |
82
+ | TL009 | Gradients never cleared, or cleared between `backward()` and `step()` |
83
+ | TL010 | Loss added to a running total or list as a tensor, keeping every graph alive |
84
+ | TL011 | `y_true` and `y_pred` swapped, or class labels passed to `roc_auc_score` |
85
+ | TL012 | Validation or test `DataLoader` with `drop_last=True` |
86
+
87
+ | Skill | Covers |
88
+ | --- | --- |
89
+ | `ml-data-leakage` | Splits, pipelines, grouped and time-ordered data, resampling, the test set |
90
+ | `ml-training-loop` | Modes, gradient order, loss inputs, shapes, seeds, mixed precision |
91
+ | `ml-metrics-eval` | Argument order, scores versus labels, averaging, imbalance, variance |
92
+ | `ml-checkpoint-resume` | What a checkpoint must hold, best versus last, verifying a resume |
93
+
94
+ ## How far to trust it
95
+
96
+ The checks parse each file with Python's `ast` module and match calls by name. There is no type
97
+ inference and no cross-file analysis, so:
98
+
99
+ - Splits are recognized by variable names such as `X_train`, `val_loader` and `y_test`.
100
+ - A bug whose cause sits in another file is not seen.
101
+ - A heuristic can be wrong. When it is, add an ignore comment and open an issue.
102
+
103
+ `python -m bench` runs the checker over a corpus of 18 buggy scripts, one or two per rule, each
104
+ with a fixed twin:
105
+
106
+ ```
107
+ rule buggy files caught false alarms on fixed twins
108
+ TL001 3 3 0
109
+ TL002 1 1 0
110
+ TL003 1 1 0
111
+ TL004 1 1 0
112
+ TL005 1 1 0
113
+ TL006 2 2 0
114
+ TL007 1 1 0
115
+ TL008 2 2 0
116
+ TL009 2 2 0
117
+ TL010 1 1 0
118
+ TL011 2 2 0
119
+ TL012 1 1 0
120
+ total 18 18 0
121
+ ```
122
+
123
+ I wrote that corpus together with the rules, so it guards against regressions and says nothing
124
+ about accuracy on code I have not seen.
125
+
126
+ For a check on someone else's code, I ran it over all 93 Python files of
127
+ [pytorch/examples](https://github.com/pytorch/examples) at commit `acc295d` and read every
128
+ finding:
129
+
130
+ ```console
131
+ git clone https://github.com/pytorch/examples && cd examples
132
+ git checkout acc295dc7b90714f1bf47f06004fc19a7fe235c4
133
+ trainlint check .
134
+ ```
135
+
136
+ It reports 21 findings. By my reading, 19 are real: 15 scripts never set a seed (one declares a
137
+ `--seed` option and never uses it), 3 tensor-parallel scripts never clear gradients, and one script
138
+ resumes from a snapshot without saving optimizer state. Two more are correct as written but
139
+ harmless, because those models have no dropout or batch norm to leave in training mode. The first
140
+ run also flagged three distributed-autograd scripts for missing `zero_grad()`, which was a false
141
+ positive that is now fixed. This is one repository of tutorial code and I did the reviewing, so
142
+ treat it as a spot check. It tells you nothing about what the rules miss.
143
+
144
+ ## Compared with other tools
145
+
146
+ | Tool | What it covers | Where trainlint differs |
147
+ | --- | --- | --- |
148
+ | ruff, pylint, flake8 | General Python style and bugs | None of their rules know about splits, eval mode or metrics. Use them alongside trainlint. |
149
+ | [TorchFix](https://github.com/meta-pytorch/torchfix) | Removed and deprecated PyTorch functions, `requires_grad` typos, unsafe `torch.load` | Its README lists no rules for training logic. trainlint does not check API deprecations. |
150
+ | [mllint](https://github.com/bvobart/mllint) | Repository-level quality of ML projects: version control, dependencies, code quality | It looks at project hygiene. trainlint reads training and evaluation code. |
151
+ | [Deepchecks](https://github.com/deepchecks/deepchecks) | Checks that run on real datasets and models, including drift | Deepchecks needs your data and model loaded. trainlint reads source only and runs nothing. |
152
+ | General skill collections | Engineering process for coding agents | They are not specific to ML. trainlint's skills target training and evaluation code and come with a checker. |
153
+
154
+ ## Roadmap
155
+
156
+ - Jupyter notebook support
157
+ - SARIF output for GitHub code scanning
158
+ - Settings in `pyproject.toml`
159
+ - More rules: training loaders without shuffling, `retain_graph` in a loop, normalization on a
160
+ whole DataFrame before the split
161
+ - A benchmark that measures how often an agent writes these bugs with and without the skills
162
+ (needs an API key, so it will be optional)
163
+
164
+ Issues labeled [good first issue](https://github.com/Satvik-jain/trainlint/labels/good%20first%20issue)
165
+ are small and have a clear definition of done.
166
+
167
+ ## Contributing
168
+
169
+ False positives and missed bugs are the most useful reports. See
170
+ [CONTRIBUTING.md](CONTRIBUTING.md) for setup and for how to add a rule.
171
+
172
+ ```console
173
+ uv sync
174
+ uv run pytest
175
+ uv run ruff check . && uv run ruff format --check .
176
+ uv run python -m bench
177
+ ```
178
+
179
+ ## License
180
+
181
+ MIT
182
+
183
+ If trainlint saved you an afternoon, a star on the repository helps other people find it.
Binary file