autotrainer 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autotrainer-0.9.0/LICENSE +21 -0
- autotrainer-0.9.0/PKG-INFO +211 -0
- autotrainer-0.9.0/README.md +156 -0
- autotrainer-0.9.0/pyproject.toml +112 -0
- autotrainer-0.9.0/setup.cfg +4 -0
- autotrainer-0.9.0/src/autotrainer/__init__.py +148 -0
- autotrainer-0.9.0/src/autotrainer/auto_optim.py +347 -0
- autotrainer-0.9.0/src/autotrainer/backends/__init__.py +0 -0
- autotrainer-0.9.0/src/autotrainer/backends/boosting_backend.py +63 -0
- autotrainer-0.9.0/src/autotrainer/backends/sklearn_backend.py +50 -0
- autotrainer-0.9.0/src/autotrainer/backends/tf_backend.py +90 -0
- autotrainer-0.9.0/src/autotrainer/backends/torch_backend.py +162 -0
- autotrainer-0.9.0/src/autotrainer/cli.py +51 -0
- autotrainer-0.9.0/src/autotrainer/detect.py +109 -0
- autotrainer-0.9.0/src/autotrainer/doctor.py +97 -0
- autotrainer-0.9.0/src/autotrainer/fitting.py +409 -0
- autotrainer-0.9.0/src/autotrainer/launcher.py +120 -0
- autotrainer-0.9.0/src/autotrainer/py.typed +0 -0
- autotrainer-0.9.0/src/autotrainer/tuning.py +215 -0
- autotrainer-0.9.0/src/autotrainer/tuning_estimator.py +206 -0
- autotrainer-0.9.0/src/autotrainer/utils.py +176 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/PKG-INFO +211 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/SOURCES.txt +36 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/dependency_links.txt +1 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/entry_points.txt +2 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/requires.txt +26 -0
- autotrainer-0.9.0/src/autotrainer.egg-info/top_level.txt +1 -0
- autotrainer-0.9.0/tests/test_auto_optim.py +206 -0
- autotrainer-0.9.0/tests/test_autotrainer.py +253 -0
- autotrainer-0.9.0/tests/test_backends.py +190 -0
- autotrainer-0.9.0/tests/test_cli.py +66 -0
- autotrainer-0.9.0/tests/test_distributed.py +146 -0
- autotrainer-0.9.0/tests/test_doctor.py +60 -0
- autotrainer-0.9.0/tests/test_fit.py +247 -0
- autotrainer-0.9.0/tests/test_launcher.py +194 -0
- autotrainer-0.9.0/tests/test_tuning.py +181 -0
- autotrainer-0.9.0/tests/test_tuning_estimator.py +108 -0
- autotrainer-0.9.0/tests/test_utils.py +113 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Suhas Goravale Siddaramu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: autotrainer
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Automatic distributed training for ML models: detects hardware, picks a strategy, tunes batch size.
|
|
5
|
+
Author: Suhas Goravale Siddaramu
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/OriAlpha/autotrainer
|
|
8
|
+
Project-URL: Repository, https://github.com/OriAlpha/autotrainer
|
|
9
|
+
Project-URL: Changelog, https://github.com/OriAlpha/autotrainer/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/OriAlpha/autotrainer/issues
|
|
11
|
+
Project-URL: Security, https://github.com/OriAlpha/autotrainer/blob/main/SECURITY.md
|
|
12
|
+
Keywords: distributed-training,pytorch,slurm,machine-learning
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Operating System :: MacOS
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
26
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
27
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
28
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
29
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
30
|
+
Classifier: Typing :: Typed
|
|
31
|
+
Requires-Python: >=3.9
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
License-File: LICENSE
|
|
34
|
+
Requires-Dist: psutil>=5.9
|
|
35
|
+
Provides-Extra: torch
|
|
36
|
+
Requires-Dist: torch>=2.0; extra == "torch"
|
|
37
|
+
Provides-Extra: sklearn
|
|
38
|
+
Requires-Dist: scikit-learn>=1.3; extra == "sklearn"
|
|
39
|
+
Provides-Extra: tf
|
|
40
|
+
Requires-Dist: tensorflow>=2.13; extra == "tf"
|
|
41
|
+
Provides-Extra: boosting
|
|
42
|
+
Requires-Dist: xgboost>=2.0; extra == "boosting"
|
|
43
|
+
Requires-Dist: lightgbm>=4.0; extra == "boosting"
|
|
44
|
+
Provides-Extra: tune
|
|
45
|
+
Requires-Dist: optuna>=3.0; extra == "tune"
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest; extra == "dev"
|
|
48
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
49
|
+
Requires-Dist: ruff; extra == "dev"
|
|
50
|
+
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
51
|
+
Requires-Dist: pdoc>=14; extra == "dev"
|
|
52
|
+
Requires-Dist: build; extra == "dev"
|
|
53
|
+
Requires-Dist: twine; extra == "dev"
|
|
54
|
+
Dynamic: license-file
|
|
55
|
+
|
|
56
|
+
# Autotrainer
|
|
57
|
+
|
|
58
|
+
[](https://github.com/OriAlpha/autotrainer/actions/workflows/ci.yml)
|
|
59
|
+
[](https://pypi.org/project/autotrainer/)
|
|
60
|
+
[](https://pypi.org/project/autotrainer/)
|
|
61
|
+
[](https://github.com/OriAlpha/autotrainer/blob/main/LICENSE)
|
|
62
|
+
[](https://github.com/astral-sh/ruff)
|
|
63
|
+
|
|
64
|
+
**Automatic distributed training and optimization for ML models.**
|
|
65
|
+
Give it a model and data - it detects your hardware (local GPUs or a SLURM
|
|
66
|
+
cluster), picks the distribution strategy, and can infer the loss function,
|
|
67
|
+
optimizer, learning rate, and schedule, or search for the best training
|
|
68
|
+
hyperparameters.
|
|
69
|
+
|
|
70
|
+
Supports **PyTorch** (DDP, SLURM multi-node), **TensorFlow/Keras**
|
|
71
|
+
(Mirrored / MultiWorker strategies), **scikit-learn**, **XGBoost**, and
|
|
72
|
+
**LightGBM** - all through one API.
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import autotrainer
|
|
76
|
+
|
|
77
|
+
# Smart defaults: loss, optimizer, LR, and schedule inferred and printed
|
|
78
|
+
model, loader, opt, loss_fn, sched = autotrainer.auto(model, loader)
|
|
79
|
+
|
|
80
|
+
# Or search for the best training recipe for YOUR model
|
|
81
|
+
best_model, best_params, study = autotrainer.tune(model, train_loader, val_loader)
|
|
82
|
+
|
|
83
|
+
# tune() also handles sklearn / XGBoost / LightGBM (curated default spaces)
|
|
84
|
+
best_est, params, study = autotrainer.tune(XGBClassifier(), (X, y), (X_val, y_val))
|
|
85
|
+
|
|
86
|
+
# Or fully hands-free: search the recipe, then train the winner to completion
|
|
87
|
+
model, params, study = autotrainer.fit(model, train_loader, val_loader)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
autotrainer run train.py # local: 1 or many GPUs, auto-detected
|
|
92
|
+
srun autotrainer run train.py # SLURM: multi-node, zero config
|
|
93
|
+
autotrainer doctor # diagnose your environment first
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Install
|
|
97
|
+
|
|
98
|
+
From PyPI:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
pip install autotrainer[torch] # just PyTorch
|
|
102
|
+
pip install autotrainer[torch,boosting] # PyTorch + XGBoost/LightGBM
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Only `psutil` is required by default; each ML framework is an opt-in extra
|
|
106
|
+
(`torch`, `sklearn`, `tf`, `boosting`, `tune`). Install only what you use.
|
|
107
|
+
|
|
108
|
+
## Install (dev mode)
|
|
109
|
+
|
|
110
|
+
Using `uv` (recommended):
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
uv venv
|
|
114
|
+
# On Windows:
|
|
115
|
+
.venv\Scripts\activate
|
|
116
|
+
# On macOS/Linux:
|
|
117
|
+
source .venv/bin/activate
|
|
118
|
+
|
|
119
|
+
uv pip install -e ".[dev,torch,sklearn,tf,boosting,tune]"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Or using standard pip:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install -e ".[dev,torch,sklearn,tf,boosting,tune]"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Use
|
|
129
|
+
|
|
130
|
+
In your training script, add one line (plus `set_epoch` at each epoch start,
|
|
131
|
+
so distributed shuffling gives every epoch a fresh order):
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
import autotrainer
|
|
135
|
+
model, loader, optimizer = autotrainer.prepare(model, loader, optimizer)
|
|
136
|
+
|
|
137
|
+
for epoch in range(epochs):
|
|
138
|
+
autotrainer.set_epoch(loader, epoch) # no-op when not distributed
|
|
139
|
+
# ... your normal training loop
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Then launch:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
autotrainer info # show what was detected
|
|
146
|
+
autotrainer run train.py # local machine (1 or many GPUs)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
On SLURM, inside your sbatch script:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
#SBATCH --nodes=2
|
|
153
|
+
#SBATCH --ntasks-per-node=4
|
|
154
|
+
#SBATCH --gres=gpu:4
|
|
155
|
+
srun autotrainer run train.py
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## One-call training: fit()
|
|
159
|
+
|
|
160
|
+
`fit()` is the whole pipeline in one call - give it a model and data, get
|
|
161
|
+
back the best model it can produce on your hardware:
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
model, params, study = autotrainer.fit(model, train_loader, val_loader, trials=30)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
1. **Tune**: Optuna searches lr / weight decay / optimizer / batch size on
|
|
168
|
+
short trials. Launched distributed, the trials are split across all
|
|
169
|
+
ranks via a shared journal-file study - one trial per process, every
|
|
170
|
+
GPU busy during the search.
|
|
171
|
+
2. **Train**: the winning recipe is retrained from your model's original
|
|
172
|
+
init through `prepare()` - so `autotrainer run` distributes it across
|
|
173
|
+
every GPU/node - with warmup+cosine, mixed precision, and early stopping
|
|
174
|
+
on the val loss. The best epoch's weights are returned.
|
|
175
|
+
|
|
176
|
+
Pass `checkpoint="fit.ckpt"` to make it preemption-safe: the full training
|
|
177
|
+
state is saved every epoch, and rerunning the same script resumes where it
|
|
178
|
+
died (skipping the search) - ideal for requeued SLURM jobs.
|
|
179
|
+
|
|
180
|
+
For long searches under multi-process launches, raise the collective
|
|
181
|
+
timeout with `AUTOTRAINER_TIMEOUT` (seconds) - see `.env.example`.
|
|
182
|
+
|
|
183
|
+
## Optional: auto batch size
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
best = autotrainer.find_batch_size(model, my_one_step_fn)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
## Documentation
|
|
190
|
+
|
|
191
|
+
- [CHANGELOG](CHANGELOG.md) - version history.
|
|
192
|
+
- [API reference](docs/README.md) - generated with pdoc; build locally with
|
|
193
|
+
`pdoc -o docs/build src/autotrainer`.
|
|
194
|
+
- [Examples](examples/) - runnable scripts for each framework and SLURM
|
|
195
|
+
`.sbatch` templates.
|
|
196
|
+
- [Environment variables](.env.example) - every knob autotrainer reads.
|
|
197
|
+
- [Contributing](CONTRIBUTING.md) | [Security policy](SECURITY.md) |
|
|
198
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
199
|
+
|
|
200
|
+
## Roadmap
|
|
201
|
+
|
|
202
|
+
- **Multi-node boosting** (xgboost.dask / lightgbm.dask across a SLURM
|
|
203
|
+
allocation) — currently single-node threads only.
|
|
204
|
+
- **Stabilization toward 1.0**: freeze the public API (mypy strict mode and
|
|
205
|
+
real 2-rank distributed tests already gate CI).
|
|
206
|
+
- **More schedulers and search spaces** beyond warmup+cosine and the default
|
|
207
|
+
Optuna recipe.
|
|
208
|
+
|
|
209
|
+
See [CHANGELOG.md](CHANGELOG.md) for the full version history, and open or
|
|
210
|
+
upvote [issues](https://github.com/OriAlpha/autotrainer/issues) to prioritize
|
|
211
|
+
these.
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Autotrainer
|
|
2
|
+
|
|
3
|
+
[](https://github.com/OriAlpha/autotrainer/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/autotrainer/)
|
|
5
|
+
[](https://pypi.org/project/autotrainer/)
|
|
6
|
+
[](https://github.com/OriAlpha/autotrainer/blob/main/LICENSE)
|
|
7
|
+
[](https://github.com/astral-sh/ruff)
|
|
8
|
+
|
|
9
|
+
**Automatic distributed training and optimization for ML models.**
|
|
10
|
+
Give it a model and data - it detects your hardware (local GPUs or a SLURM
|
|
11
|
+
cluster), picks the distribution strategy, and can infer the loss function,
|
|
12
|
+
optimizer, learning rate, and schedule, or search for the best training
|
|
13
|
+
hyperparameters.
|
|
14
|
+
|
|
15
|
+
Supports **PyTorch** (DDP, SLURM multi-node), **TensorFlow/Keras**
|
|
16
|
+
(Mirrored / MultiWorker strategies), **scikit-learn**, **XGBoost**, and
|
|
17
|
+
**LightGBM** - all through one API.
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import autotrainer
|
|
21
|
+
|
|
22
|
+
# Smart defaults: loss, optimizer, LR, and schedule inferred and printed
|
|
23
|
+
model, loader, opt, loss_fn, sched = autotrainer.auto(model, loader)
|
|
24
|
+
|
|
25
|
+
# Or search for the best training recipe for YOUR model
|
|
26
|
+
best_model, best_params, study = autotrainer.tune(model, train_loader, val_loader)
|
|
27
|
+
|
|
28
|
+
# tune() also handles sklearn / XGBoost / LightGBM (curated default spaces)
|
|
29
|
+
best_est, params, study = autotrainer.tune(XGBClassifier(), (X, y), (X_val, y_val))
|
|
30
|
+
|
|
31
|
+
# Or fully hands-free: search the recipe, then train the winner to completion
|
|
32
|
+
model, params, study = autotrainer.fit(model, train_loader, val_loader)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
autotrainer run train.py # local: 1 or many GPUs, auto-detected
|
|
37
|
+
srun autotrainer run train.py # SLURM: multi-node, zero config
|
|
38
|
+
autotrainer doctor # diagnose your environment first
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
From PyPI:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install autotrainer[torch] # just PyTorch
|
|
47
|
+
pip install autotrainer[torch,boosting] # PyTorch + XGBoost/LightGBM
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Only `psutil` is required by default; each ML framework is an opt-in extra
|
|
51
|
+
(`torch`, `sklearn`, `tf`, `boosting`, `tune`). Install only what you use.
|
|
52
|
+
|
|
53
|
+
## Install (dev mode)
|
|
54
|
+
|
|
55
|
+
Using `uv` (recommended):
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv venv
|
|
59
|
+
# On Windows:
|
|
60
|
+
.venv\Scripts\activate
|
|
61
|
+
# On macOS/Linux:
|
|
62
|
+
source .venv/bin/activate
|
|
63
|
+
|
|
64
|
+
uv pip install -e ".[dev,torch,sklearn,tf,boosting,tune]"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or using standard pip:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install -e ".[dev,torch,sklearn,tf,boosting,tune]"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Use
|
|
74
|
+
|
|
75
|
+
In your training script, add one line (plus `set_epoch` at each epoch start,
|
|
76
|
+
so distributed shuffling gives every epoch a fresh order):
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
import autotrainer
|
|
80
|
+
model, loader, optimizer = autotrainer.prepare(model, loader, optimizer)
|
|
81
|
+
|
|
82
|
+
for epoch in range(epochs):
|
|
83
|
+
autotrainer.set_epoch(loader, epoch) # no-op when not distributed
|
|
84
|
+
# ... your normal training loop
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Then launch:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
autotrainer info # show what was detected
|
|
91
|
+
autotrainer run train.py # local machine (1 or many GPUs)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
On SLURM, inside your sbatch script:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
#SBATCH --nodes=2
|
|
98
|
+
#SBATCH --ntasks-per-node=4
|
|
99
|
+
#SBATCH --gres=gpu:4
|
|
100
|
+
srun autotrainer run train.py
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## One-call training: fit()
|
|
104
|
+
|
|
105
|
+
`fit()` is the whole pipeline in one call - give it a model and data, get
|
|
106
|
+
back the best model it can produce on your hardware:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
model, params, study = autotrainer.fit(model, train_loader, val_loader, trials=30)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
1. **Tune**: Optuna searches lr / weight decay / optimizer / batch size on
|
|
113
|
+
short trials. Launched distributed, the trials are split across all
|
|
114
|
+
ranks via a shared journal-file study - one trial per process, every
|
|
115
|
+
GPU busy during the search.
|
|
116
|
+
2. **Train**: the winning recipe is retrained from your model's original
|
|
117
|
+
init through `prepare()` - so `autotrainer run` distributes it across
|
|
118
|
+
every GPU/node - with warmup+cosine, mixed precision, and early stopping
|
|
119
|
+
on the val loss. The best epoch's weights are returned.
|
|
120
|
+
|
|
121
|
+
Pass `checkpoint="fit.ckpt"` to make it preemption-safe: the full training
|
|
122
|
+
state is saved every epoch, and rerunning the same script resumes where it
|
|
123
|
+
died (skipping the search) - ideal for requeued SLURM jobs.
|
|
124
|
+
|
|
125
|
+
For long searches under multi-process launches, raise the collective
|
|
126
|
+
timeout with `AUTOTRAINER_TIMEOUT` (seconds) - see `.env.example`.
|
|
127
|
+
|
|
128
|
+
## Optional: auto batch size
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
best = autotrainer.find_batch_size(model, my_one_step_fn)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Documentation
|
|
135
|
+
|
|
136
|
+
- [CHANGELOG](CHANGELOG.md) - version history.
|
|
137
|
+
- [API reference](docs/README.md) - generated with pdoc; build locally with
|
|
138
|
+
`pdoc -o docs/build src/autotrainer`.
|
|
139
|
+
- [Examples](examples/) - runnable scripts for each framework and SLURM
|
|
140
|
+
`.sbatch` templates.
|
|
141
|
+
- [Environment variables](.env.example) - every knob autotrainer reads.
|
|
142
|
+
- [Contributing](CONTRIBUTING.md) | [Security policy](SECURITY.md) |
|
|
143
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
144
|
+
|
|
145
|
+
## Roadmap
|
|
146
|
+
|
|
147
|
+
- **Multi-node boosting** (xgboost.dask / lightgbm.dask across a SLURM
|
|
148
|
+
allocation) — currently single-node threads only.
|
|
149
|
+
- **Stabilization toward 1.0**: freeze the public API (mypy strict mode and
|
|
150
|
+
real 2-rank distributed tests already gate CI).
|
|
151
|
+
- **More schedulers and search spaces** beyond warmup+cosine and the default
|
|
152
|
+
Optuna recipe.
|
|
153
|
+
|
|
154
|
+
See [CHANGELOG.md](CHANGELOG.md) for the full version history, and open or
|
|
155
|
+
upvote [issues](https://github.com/OriAlpha/autotrainer/issues) to prioritize
|
|
156
|
+
these.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "autotrainer"
|
|
7
|
+
version = "0.9.0"
|
|
8
|
+
description = "Automatic distributed training for ML models: detects hardware, picks a strategy, tunes batch size."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Suhas Goravale Siddaramu" }]
|
|
14
|
+
keywords = ["distributed-training", "pytorch", "slurm", "machine-learning"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Operating System :: MacOS",
|
|
21
|
+
"Operating System :: Microsoft :: Windows",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
29
|
+
"Programming Language :: Python :: Implementation :: PyPy",
|
|
30
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
32
|
+
"Topic :: System :: Distributed Computing",
|
|
33
|
+
"Typing :: Typed",
|
|
34
|
+
]
|
|
35
|
+
dependencies = [
|
|
36
|
+
"psutil>=5.9",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
torch = ["torch>=2.0"]
|
|
41
|
+
sklearn = ["scikit-learn>=1.3"]
|
|
42
|
+
tf = ["tensorflow>=2.13"]
|
|
43
|
+
boosting = ["xgboost>=2.0", "lightgbm>=4.0"]
|
|
44
|
+
tune = ["optuna>=3.0"]
|
|
45
|
+
dev = ["pytest", "pytest-cov>=4.1", "ruff", "mypy>=1.8", "pdoc>=14", "build", "twine"]
|
|
46
|
+
|
|
47
|
+
[project.urls]
|
|
48
|
+
Homepage = "https://github.com/OriAlpha/autotrainer"
|
|
49
|
+
Repository = "https://github.com/OriAlpha/autotrainer"
|
|
50
|
+
Changelog = "https://github.com/OriAlpha/autotrainer/blob/main/CHANGELOG.md"
|
|
51
|
+
Issues = "https://github.com/OriAlpha/autotrainer/issues"
|
|
52
|
+
Security = "https://github.com/OriAlpha/autotrainer/blob/main/SECURITY.md"
|
|
53
|
+
|
|
54
|
+
[project.scripts]
|
|
55
|
+
autotrainer = "autotrainer.cli:main"
|
|
56
|
+
|
|
57
|
+
[tool.setuptools.packages.find]
|
|
58
|
+
where = ["src"]
|
|
59
|
+
|
|
60
|
+
# Ship the PEP 561 marker so downstream type checkers pick up our annotations.
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
autotrainer = ["py.typed"]
|
|
63
|
+
|
|
64
|
+
[tool.ruff]
|
|
65
|
+
line-length = 100
|
|
66
|
+
target-version = "py39"
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint]
|
|
69
|
+
# E/W = pycodestyle, F = pyflakes, I = isort, UP = pyupgrade,
|
|
70
|
+
# B = bugbear, C4 = comprehensions, SIM = simplify
|
|
71
|
+
select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM"]
|
|
72
|
+
# Module-level lazy imports of optional frameworks are intentional; the
|
|
73
|
+
# dispatcher defers torch/tf/sklearn/etc. to call time so the base install
|
|
74
|
+
# needs no ML framework.
|
|
75
|
+
ignore = ["E402"]
|
|
76
|
+
|
|
77
|
+
[tool.ruff.lint.per-file-ignores]
|
|
78
|
+
"tests/*" = ["E501"]
|
|
79
|
+
|
|
80
|
+
[tool.ruff.format]
|
|
81
|
+
quote-style = "double"
|
|
82
|
+
|
|
83
|
+
[tool.mypy]
|
|
84
|
+
# mypy 2.x dropped 3.9 as a target; 3.11 is in the CI matrix. Code uses
|
|
85
|
+
# `from __future__ import annotations`, so `X | None` syntax stays 3.9-safe.
|
|
86
|
+
# (ruff's target-version = "py39" enforces 3.9 runtime syntax separately.)
|
|
87
|
+
python_version = "3.11"
|
|
88
|
+
# torch/tf/sklearn/xgboost/lightgbm/optuna are optional extras; don't fail
|
|
89
|
+
# the typecheck when they're absent. Skip following into their (and numpy's)
|
|
90
|
+
# bundled stubs so an unrelated stub syntax error can't block our check.
|
|
91
|
+
ignore_missing_imports = true
|
|
92
|
+
follow_imports = "skip"
|
|
93
|
+
# Strict mode (roadmap item for 1.0). With follow_imports = "skip" the
|
|
94
|
+
# optional-framework objects are Any, so strictness mostly polices OUR
|
|
95
|
+
# signatures and generics rather than torch/optuna internals.
|
|
96
|
+
strict = true
|
|
97
|
+
|
|
98
|
+
[tool.coverage.run]
|
|
99
|
+
source = ["src/autotrainer"]
|
|
100
|
+
branch = true
|
|
101
|
+
|
|
102
|
+
[tool.coverage.report]
|
|
103
|
+
# Set after measuring actual coverage with the expanded suite. Leave at 0
|
|
104
|
+
# until the suite lands, then raise to the achieved floor.
|
|
105
|
+
fail_under = 75
|
|
106
|
+
show_missing = true
|
|
107
|
+
skip_covered = false
|
|
108
|
+
|
|
109
|
+
[tool.pytest.ini_options]
|
|
110
|
+
# Coverage is computed on every run; the XML is uploaded as a CI artifact.
|
|
111
|
+
addopts = "--cov=autotrainer --cov-report=term-missing --cov-report=xml"
|
|
112
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""autotrainer: automatic distributed training.
|
|
2
|
+
|
|
3
|
+
Usage inside a training script (one line before the loop):
|
|
4
|
+
|
|
5
|
+
import autotrainer
|
|
6
|
+
model, loader, opt = autotrainer.prepare(model, loader, opt)
|
|
7
|
+
|
|
8
|
+
Launch with:
|
|
9
|
+
|
|
10
|
+
autotrainer run train.py # local: auto-detects GPUs
|
|
11
|
+
srun autotrainer run train.py # inside an sbatch script on SLURM
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
__version__ = "0.9.0"
|
|
19
|
+
|
|
20
|
+
from .utils import ( # noqa: E402,F401
|
|
21
|
+
GradScaler,
|
|
22
|
+
autocast_context,
|
|
23
|
+
barrier,
|
|
24
|
+
is_main,
|
|
25
|
+
print0,
|
|
26
|
+
rank,
|
|
27
|
+
save0,
|
|
28
|
+
set_epoch,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def scope() -> Any:
|
|
33
|
+
"""TensorFlow: strategy scope. Create and compile your model inside it."""
|
|
34
|
+
from .backends.tf_backend import scope as _s
|
|
35
|
+
|
|
36
|
+
return _s()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def scale_batch_size(per_replica_batch: int) -> int:
|
|
40
|
+
"""TensorFlow: convert a per-replica batch size to the global batch size."""
|
|
41
|
+
from .backends.tf_backend import scale_batch_size as _s
|
|
42
|
+
|
|
43
|
+
return _s(per_replica_batch)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def boost_params(params: dict[str, Any] | None = None, lib: str = "xgboost") -> dict[str, Any]:
|
|
47
|
+
"""XGBoost/LightGBM native API: params dict with auto thread count."""
|
|
48
|
+
from .backends.boosting_backend import boost_params as _b
|
|
49
|
+
|
|
50
|
+
return _b(params, lib=lib)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def prepare(model: Any, dataloader: Any = None, optimizer: Any = None) -> Any:
|
|
54
|
+
"""Framework dispatcher: route to the right backend by model type."""
|
|
55
|
+
mod = type(model).__module__ or ""
|
|
56
|
+
|
|
57
|
+
if mod.startswith("torch") or _is_torch_module(model):
|
|
58
|
+
from .backends.torch_backend import prepare as torch_prepare
|
|
59
|
+
|
|
60
|
+
return torch_prepare(model, dataloader, optimizer)
|
|
61
|
+
|
|
62
|
+
if mod.startswith(("keras", "tensorflow")):
|
|
63
|
+
raise TypeError(
|
|
64
|
+
"TensorFlow models must be created inside `with autotrainer.scope():` "
|
|
65
|
+
"rather than passed to prepare() - see README."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# Must come before the sklearn check: XGB/LGBM sklearn-API models
|
|
69
|
+
# subclass BaseEstimator and would be misrouted otherwise.
|
|
70
|
+
if mod.startswith(("xgboost", "lightgbm")):
|
|
71
|
+
from .backends.boosting_backend import prepare as boost_prepare
|
|
72
|
+
|
|
73
|
+
return boost_prepare(model)
|
|
74
|
+
|
|
75
|
+
if mod.startswith("sklearn") or _is_sklearn_estimator(model):
|
|
76
|
+
from .backends.sklearn_backend import prepare as sklearn_prepare
|
|
77
|
+
|
|
78
|
+
return sklearn_prepare(model)
|
|
79
|
+
|
|
80
|
+
raise TypeError(f"Unrecognized model type: {type(model)!r}")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def find_batch_size(model: Any, sample_batch_fn: Any, start: int = 2, max_bs: int = 4096) -> int:
|
|
84
|
+
from .backends.torch_backend import find_batch_size as _f
|
|
85
|
+
|
|
86
|
+
return _f(model, sample_batch_fn, start=start, max_bs=max_bs)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def auto(model: Any, dataloader: Any, **kwargs: Any) -> tuple[Any, ...]:
|
|
90
|
+
"""PyTorch: infer loss, optimizer, LR, and schedule, then distribute."""
|
|
91
|
+
from .auto_optim import auto as _a
|
|
92
|
+
|
|
93
|
+
return _a(model, dataloader, **kwargs)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def find_lr(model: Any, dataloader: Any, loss_fn: Any, **kwargs: Any) -> float:
|
|
97
|
+
"""PyTorch: LR range test on a throwaway model copy."""
|
|
98
|
+
from .auto_optim import find_lr as _f
|
|
99
|
+
|
|
100
|
+
return _f(model, dataloader, loss_fn, **kwargs)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def tune(model: Any, train_loader: Any, val_loader: Any, **kwargs: Any) -> tuple[Any, ...]:
|
|
104
|
+
"""Search hyperparameters for a model (dispatches by framework).
|
|
105
|
+
|
|
106
|
+
PyTorch modules: pass DataLoaders; searches the training recipe
|
|
107
|
+
(lr, weight decay, optimizer, batch size).
|
|
108
|
+
sklearn-API estimators (scikit-learn, XGBoost, LightGBM): pass
|
|
109
|
+
``(X, y)`` tuples; searches the model's hyperparameters, with curated
|
|
110
|
+
default spaces for the common families.
|
|
111
|
+
"""
|
|
112
|
+
mod = type(model).__module__ or ""
|
|
113
|
+
if mod.startswith("torch") or _is_torch_module(model):
|
|
114
|
+
from .tuning import tune as _t
|
|
115
|
+
|
|
116
|
+
return _t(model, train_loader, val_loader, **kwargs)
|
|
117
|
+
if mod.startswith(("xgboost", "lightgbm", "sklearn")) or _is_sklearn_estimator(model):
|
|
118
|
+
from .tuning_estimator import tune_estimator as _te
|
|
119
|
+
|
|
120
|
+
return _te(model, train_loader, val_loader, **kwargs)
|
|
121
|
+
raise TypeError(
|
|
122
|
+
f"tune() supports PyTorch modules and sklearn-API estimators, got {type(model)!r}."
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def fit(model: Any, train_loader: Any, val_loader: Any, **kwargs: Any) -> tuple[Any, ...]:
|
|
127
|
+
"""PyTorch: tune the recipe, then fully train the winner (with early stopping)."""
|
|
128
|
+
from .fitting import fit as _f
|
|
129
|
+
|
|
130
|
+
return _f(model, train_loader, val_loader, **kwargs)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _is_torch_module(model: Any) -> bool:
|
|
134
|
+
try:
|
|
135
|
+
import torch # noqa: PLC0415
|
|
136
|
+
|
|
137
|
+
return isinstance(model, torch.nn.Module)
|
|
138
|
+
except ImportError:
|
|
139
|
+
return False
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _is_sklearn_estimator(model: Any) -> bool:
|
|
143
|
+
try:
|
|
144
|
+
from sklearn.base import BaseEstimator # noqa: PLC0415
|
|
145
|
+
|
|
146
|
+
return isinstance(model, BaseEstimator)
|
|
147
|
+
except ImportError:
|
|
148
|
+
return False
|