coqui-tts-trainer 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. coqui-tts-trainer-0.1.0/MANIFEST.in +12 -0
  2. coqui-tts-trainer-0.1.0/PKG-INFO +222 -0
  3. coqui-tts-trainer-0.1.0/README.md +166 -0
  4. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/PKG-INFO +222 -0
  5. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/SOURCES.txt +44 -0
  6. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/dependency_links.txt +1 -0
  7. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/not-zip-safe +1 -0
  8. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/requires.txt +32 -0
  9. coqui-tts-trainer-0.1.0/coqui_tts_trainer.egg-info/top_level.txt +1 -0
  10. coqui-tts-trainer-0.1.0/pyproject.toml +32 -0
  11. coqui-tts-trainer-0.1.0/requirements.dev.txt +6 -0
  12. coqui-tts-trainer-0.1.0/requirements.test.txt +1 -0
  13. coqui-tts-trainer-0.1.0/requirements.txt +6 -0
  14. coqui-tts-trainer-0.1.0/setup.cfg +13 -0
  15. coqui-tts-trainer-0.1.0/setup.py +119 -0
  16. coqui-tts-trainer-0.1.0/tests/test_continue_train.py +32 -0
  17. coqui-tts-trainer-0.1.0/tests/test_lr_schedulers.py +41 -0
  18. coqui-tts-trainer-0.1.0/tests/test_num_gpus.py +65 -0
  19. coqui-tts-trainer-0.1.0/tests/test_train_batch_size_finder.py +23 -0
  20. coqui-tts-trainer-0.1.0/tests/test_train_gan.py +618 -0
  21. coqui-tts-trainer-0.1.0/tests/test_train_mnist.py +23 -0
  22. coqui-tts-trainer-0.1.0/trainer/VERSION +1 -0
  23. coqui-tts-trainer-0.1.0/trainer/__init__.py +9 -0
  24. coqui-tts-trainer-0.1.0/trainer/callbacks.py +217 -0
  25. coqui-tts-trainer-0.1.0/trainer/distribute.py +73 -0
  26. coqui-tts-trainer-0.1.0/trainer/generic_utils.py +148 -0
  27. coqui-tts-trainer-0.1.0/trainer/io.py +327 -0
  28. coqui-tts-trainer-0.1.0/trainer/logger.py +8 -0
  29. coqui-tts-trainer-0.1.0/trainer/logging/__init__.py +68 -0
  30. coqui-tts-trainer-0.1.0/trainer/logging/aim_logger.py +166 -0
  31. coqui-tts-trainer-0.1.0/trainer/logging/base_dash_logger.py +89 -0
  32. coqui-tts-trainer-0.1.0/trainer/logging/clearml_logger.py +69 -0
  33. coqui-tts-trainer-0.1.0/trainer/logging/console_logger.py +115 -0
  34. coqui-tts-trainer-0.1.0/trainer/logging/dummy_logger.py +72 -0
  35. coqui-tts-trainer-0.1.0/trainer/logging/mlflow_logger.py +148 -0
  36. coqui-tts-trainer-0.1.0/trainer/logging/tensorboard_logger.py +71 -0
  37. coqui-tts-trainer-0.1.0/trainer/logging/wandb_logger.py +100 -0
  38. coqui-tts-trainer-0.1.0/trainer/model.py +188 -0
  39. coqui-tts-trainer-0.1.0/trainer/torch.py +117 -0
  40. coqui-tts-trainer-0.1.0/trainer/trainer.py +2165 -0
  41. coqui-tts-trainer-0.1.0/trainer/trainer_utils.py +170 -0
  42. coqui-tts-trainer-0.1.0/trainer/utils/__init__.py +0 -0
  43. coqui-tts-trainer-0.1.0/trainer/utils/cpu_memory.py +40 -0
  44. coqui-tts-trainer-0.1.0/trainer/utils/cuda_memory.py +98 -0
  45. coqui-tts-trainer-0.1.0/trainer/utils/distributed.py +71 -0
@@ -0,0 +1,12 @@
1
+ include README.md
2
+ include LICENSE.txt
3
+ include requirements.*.txt
4
+ include requirements.txt
5
+ include trainer/VERSION
6
+ recursive-include trainer *.json
7
+ recursive-include trainer *.html
8
+ recursive-include trainer *.png
9
+ recursive-include trainer *.md
10
+ recursive-include trainer *.py
11
+ recursive-include trainer *.pyx
12
+ recursive-include images *.png
@@ -0,0 +1,222 @@
1
+ Metadata-Version: 2.1
2
+ Name: coqui-tts-trainer
3
+ Version: 0.1.0
4
+ Summary: General purpose model trainer for PyTorch that is more flexible than it should be, by 🐸Coqui.
5
+ Home-page: https://github.com/idiap/coqui-ai-Trainer
6
+ Author: Eren Gölge
7
+ Author-email: egolge@coqui.ai
8
+ Maintainer: Enno Hermann
9
+ Maintainer-email: enno.hermann@gmail.com
10
+ License: Apache2
11
+ Project-URL: Documentation, https://github.com/idiap/coqui-ai-Trainer
12
+ Project-URL: Tracker, https://github.com/idiap/coqui-ai-Trainer/issues
13
+ Project-URL: Repository, https://github.com/idiap/coqui-ai-Trainer
14
+ Classifier: Environment :: Console
15
+ Classifier: Natural Language :: English
16
+ Classifier: Development Status :: 3 - Alpha
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: License :: OSI Approved :: Apache Software License
19
+ Classifier: Operating System :: OS Independent
20
+ Classifier: Programming Language :: Python :: 3.8
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Requires-Python: >=3.6.0, <3.13
26
+ Description-Content-Type: text/markdown
27
+ Requires-Dist: torch>=1.7
28
+ Requires-Dist: coqpit
29
+ Requires-Dist: psutil
30
+ Requires-Dist: fsspec
31
+ Requires-Dist: tensorboard
32
+ Requires-Dist: soundfile
33
+ Provides-Extra: dev
34
+ Requires-Dist: black; extra == "dev"
35
+ Requires-Dist: coverage; extra == "dev"
36
+ Requires-Dist: isort; extra == "dev"
37
+ Requires-Dist: pytest; extra == "dev"
38
+ Requires-Dist: pylint; extra == "dev"
39
+ Requires-Dist: accelerate; extra == "dev"
40
+ Provides-Extra: test
41
+ Requires-Dist: torchvision; extra == "test"
42
+ Provides-Extra: all
43
+ Requires-Dist: torch>=1.7; extra == "all"
44
+ Requires-Dist: coqpit; extra == "all"
45
+ Requires-Dist: psutil; extra == "all"
46
+ Requires-Dist: fsspec; extra == "all"
47
+ Requires-Dist: tensorboard; extra == "all"
48
+ Requires-Dist: soundfile; extra == "all"
49
+ Requires-Dist: black; extra == "all"
50
+ Requires-Dist: coverage; extra == "all"
51
+ Requires-Dist: isort; extra == "all"
52
+ Requires-Dist: pytest; extra == "all"
53
+ Requires-Dist: pylint; extra == "all"
54
+ Requires-Dist: accelerate; extra == "all"
55
+ Requires-Dist: torchvision; extra == "all"
56
+
57
+ <p align="center"><img src="https://user-images.githubusercontent.com/1402048/151947958-0bcadf38-3a82-4b4e-96b4-a38d3721d737.png" align="right" height="255px" /></p>
58
+
59
+ # 👟 Trainer
60
+ An opinionated general purpose model trainer on PyTorch with a simple code base.
61
+ Fork of the [original, unmaintained repository](https://github.com/coqui-ai/Trainer). New PyPI package: [coqui-tts-trainer](https://pypi.org/project/coqui-tts-trainer)
62
+
63
+ ## Installation
64
+
65
+ From Github:
66
+
67
+ ```console
68
+ git clone https://github.com/idiap/coqui-ai-Trainer
69
+ cd coqui-ai-Trainer
70
+ make install
71
+ ```
72
+
73
+ From PyPI:
74
+
75
+ ```console
76
+ pip install coqui-tts-trainer
77
+ ```
78
+
79
+ ## Implementing a model
80
+ Subclass and overload the functions in the [```TrainerModel()```](trainer/model.py)
81
+
82
+
83
+ ## Training a model with auto-optimization
84
+ See the [MNIST example](examples/train_mnist.py).
85
+
86
+
87
+ ## Training a model with advanced optimization
88
+ With 👟 you can define the whole optimization cycle as you want as the in GAN example below. It enables more
89
+ under-the-hood control and flexibility for more advanced training loops.
90
+
91
+ You just have to use the ```scaled_backward()``` function to handle mixed precision training.
92
+
93
+ ```python
94
+ ...
95
+
96
+ def optimize(self, batch, trainer):
97
+ imgs, _ = batch
98
+
99
+ # sample noise
100
+ z = torch.randn(imgs.shape[0], 100)
101
+ z = z.type_as(imgs)
102
+
103
+ # train discriminator
104
+ imgs_gen = self.generator(z)
105
+ logits = self.discriminator(imgs_gen.detach())
106
+ fake = torch.zeros(imgs.size(0), 1)
107
+ fake = fake.type_as(imgs)
108
+ loss_fake = trainer.criterion(logits, fake)
109
+
110
+ valid = torch.ones(imgs.size(0), 1)
111
+ valid = valid.type_as(imgs)
112
+ logits = self.discriminator(imgs)
113
+ loss_real = trainer.criterion(logits, valid)
114
+ loss_disc = (loss_real + loss_fake) / 2
115
+
116
+ # step dicriminator
117
+ _, _ = self.scaled_backward(loss_disc, None, trainer, trainer.optimizer[0])
118
+
119
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
120
+ trainer.optimizer[0].step()
121
+ trainer.optimizer[0].zero_grad()
122
+
123
+ # train generator
124
+ imgs_gen = self.generator(z)
125
+
126
+ valid = torch.ones(imgs.size(0), 1)
127
+ valid = valid.type_as(imgs)
128
+
129
+ logits = self.discriminator(imgs_gen)
130
+ loss_gen = trainer.criterion(logits, valid)
131
+
132
+ # step generator
133
+ _, _ = self.scaled_backward(loss_gen, None, trainer, trainer.optimizer[1])
134
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
135
+ trainer.optimizer[1].step()
136
+ trainer.optimizer[1].zero_grad()
137
+ return {"model_outputs": logits}, {"loss_gen": loss_gen, "loss_disc": loss_disc}
138
+
139
+ ...
140
+ ```
141
+
142
+ See the [GAN training example](examples/train_simple_gan.py) with Gradient Accumulation
143
+
144
+
145
+ ## Training with Batch Size Finder
146
+ see the test script [here](tests/test_train_batch_size_finder.py) for training with batch size finder.
147
+
148
+
149
+ The batch size finder starts at a default BS(defaults to 2048 but can also be user defined) and searches for the largest batch size that can fit on your hardware. you should expect for it to run multiple trainings until it finds it. to use it instead of calling ```trainer.fit()``` youll call ```trainer.fit_with_largest_batch_size(starting_batch_size=2048)``` with ```starting_batch_size``` being the batch the size you want to start the search with. very useful if you are wanting to use as much gpu mem as possible.
150
+
151
+ ## Training with DDP
152
+
153
+ ```console
154
+ $ python -m trainer.distribute --script path/to/your/train.py --gpus "0,1"
155
+ ```
156
+
157
+ We don't use ```.spawn()``` to initiate multi-gpu training since it causes certain limitations.
158
+
159
+ - Everything must the pickable.
160
+ - ```.spawn()``` trains the model in subprocesses and the model in the main process is not updated.
161
+ - DataLoader with N processes gets really slow when the N is large.
162
+
163
+ ## Training with [Accelerate](https://huggingface.co/docs/accelerate/index)
164
+
165
+ Setting `use_accelerate` in `TrainingArgs` to `True` will enable training with Accelerate.
166
+
167
+ You can also use it for multi-gpu or distributed training.
168
+
169
+ ```console
170
+ CUDA_VISIBLE_DEVICES="0,1,2" accelerate launch --multi_gpu --num_processes 3 train_recipe_autoregressive_prompt.py
171
+ ```
172
+
173
+ See the [Accelerate docs](https://huggingface.co/docs/accelerate/basic_tutorials/launch).
174
+
175
+ ## Adding a callback
176
+ 👟 Supports callbacks to customize your runs. You can either set callbacks in your model implementations or give them
177
+ explicitly to the Trainer.
178
+
179
+ Please check `trainer.utils.callbacks` to see available callbacks.
180
+
181
+ Here is how you provide an explicit call back to a 👟Trainer object for weight reinitialization.
182
+
183
+ ```python
184
+ def my_callback(trainer):
185
+ print(" > My callback was called.")
186
+
187
+ trainer = Trainer(..., callbacks={"on_init_end": my_callback})
188
+ trainer.fit()
189
+ ```
190
+
191
+ ## Profiling example
192
+
193
+ - Create the torch profiler as you like and pass it to the trainer.
194
+ ```python
195
+ import torch
196
+ profiler = torch.profiler.profile(
197
+ activities=[
198
+ torch.profiler.ProfilerActivity.CPU,
199
+ torch.profiler.ProfilerActivity.CUDA,
200
+ ],
201
+ schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
202
+ on_trace_ready=torch.profiler.tensorboard_trace_handler("./profiler/"),
203
+ record_shapes=True,
204
+ profile_memory=True,
205
+ with_stack=True,
206
+ )
207
+ prof = trainer.profile_fit(profiler, epochs=1, small_run=64)
208
+ then run Tensorboard
209
+ ```
210
+ - Run the tensorboard.
211
+ ```console
212
+ tensorboard --logdir="./profiler/"
213
+ ```
214
+
215
+ ## Supported Experiment Loggers
216
+ - [Tensorboard](https://www.tensorflow.org/tensorboard) - actively maintained
217
+ - [ClearML](https://clear.ml/) - actively maintained
218
+ - [MLFlow](https://mlflow.org/)
219
+ - [Aim](https://aimstack.io/)
220
+ - [WandDB](https://wandb.ai/)
221
+
222
+ To add a new logger, you must subclass [BaseDashboardLogger](trainer/logging/base_dash_logger.py) and overload its functions.
@@ -0,0 +1,166 @@
1
+ <p align="center"><img src="https://user-images.githubusercontent.com/1402048/151947958-0bcadf38-3a82-4b4e-96b4-a38d3721d737.png" align="right" height="255px" /></p>
2
+
3
+ # 👟 Trainer
4
+ An opinionated general purpose model trainer on PyTorch with a simple code base.
5
+ Fork of the [original, unmaintained repository](https://github.com/coqui-ai/Trainer). New PyPI package: [coqui-tts-trainer](https://pypi.org/project/coqui-tts-trainer)
6
+
7
+ ## Installation
8
+
9
+ From Github:
10
+
11
+ ```console
12
+ git clone https://github.com/idiap/coqui-ai-Trainer
13
+ cd coqui-ai-Trainer
14
+ make install
15
+ ```
16
+
17
+ From PyPI:
18
+
19
+ ```console
20
+ pip install coqui-tts-trainer
21
+ ```
22
+
23
+ ## Implementing a model
24
+ Subclass and overload the functions in the [```TrainerModel()```](trainer/model.py)
25
+
26
+
27
+ ## Training a model with auto-optimization
28
+ See the [MNIST example](examples/train_mnist.py).
29
+
30
+
31
+ ## Training a model with advanced optimization
32
+ With 👟 you can define the whole optimization cycle as you want as the in GAN example below. It enables more
33
+ under-the-hood control and flexibility for more advanced training loops.
34
+
35
+ You just have to use the ```scaled_backward()``` function to handle mixed precision training.
36
+
37
+ ```python
38
+ ...
39
+
40
+ def optimize(self, batch, trainer):
41
+ imgs, _ = batch
42
+
43
+ # sample noise
44
+ z = torch.randn(imgs.shape[0], 100)
45
+ z = z.type_as(imgs)
46
+
47
+ # train discriminator
48
+ imgs_gen = self.generator(z)
49
+ logits = self.discriminator(imgs_gen.detach())
50
+ fake = torch.zeros(imgs.size(0), 1)
51
+ fake = fake.type_as(imgs)
52
+ loss_fake = trainer.criterion(logits, fake)
53
+
54
+ valid = torch.ones(imgs.size(0), 1)
55
+ valid = valid.type_as(imgs)
56
+ logits = self.discriminator(imgs)
57
+ loss_real = trainer.criterion(logits, valid)
58
+ loss_disc = (loss_real + loss_fake) / 2
59
+
60
+ # step dicriminator
61
+ _, _ = self.scaled_backward(loss_disc, None, trainer, trainer.optimizer[0])
62
+
63
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
64
+ trainer.optimizer[0].step()
65
+ trainer.optimizer[0].zero_grad()
66
+
67
+ # train generator
68
+ imgs_gen = self.generator(z)
69
+
70
+ valid = torch.ones(imgs.size(0), 1)
71
+ valid = valid.type_as(imgs)
72
+
73
+ logits = self.discriminator(imgs_gen)
74
+ loss_gen = trainer.criterion(logits, valid)
75
+
76
+ # step generator
77
+ _, _ = self.scaled_backward(loss_gen, None, trainer, trainer.optimizer[1])
78
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
79
+ trainer.optimizer[1].step()
80
+ trainer.optimizer[1].zero_grad()
81
+ return {"model_outputs": logits}, {"loss_gen": loss_gen, "loss_disc": loss_disc}
82
+
83
+ ...
84
+ ```
85
+
86
+ See the [GAN training example](examples/train_simple_gan.py) with Gradient Accumulation
87
+
88
+
89
+ ## Training with Batch Size Finder
90
+ see the test script [here](tests/test_train_batch_size_finder.py) for training with batch size finder.
91
+
92
+
93
+ The batch size finder starts at a default BS(defaults to 2048 but can also be user defined) and searches for the largest batch size that can fit on your hardware. you should expect for it to run multiple trainings until it finds it. to use it instead of calling ```trainer.fit()``` youll call ```trainer.fit_with_largest_batch_size(starting_batch_size=2048)``` with ```starting_batch_size``` being the batch the size you want to start the search with. very useful if you are wanting to use as much gpu mem as possible.
94
+
95
+ ## Training with DDP
96
+
97
+ ```console
98
+ $ python -m trainer.distribute --script path/to/your/train.py --gpus "0,1"
99
+ ```
100
+
101
+ We don't use ```.spawn()``` to initiate multi-gpu training since it causes certain limitations.
102
+
103
+ - Everything must the pickable.
104
+ - ```.spawn()``` trains the model in subprocesses and the model in the main process is not updated.
105
+ - DataLoader with N processes gets really slow when the N is large.
106
+
107
+ ## Training with [Accelerate](https://huggingface.co/docs/accelerate/index)
108
+
109
+ Setting `use_accelerate` in `TrainingArgs` to `True` will enable training with Accelerate.
110
+
111
+ You can also use it for multi-gpu or distributed training.
112
+
113
+ ```console
114
+ CUDA_VISIBLE_DEVICES="0,1,2" accelerate launch --multi_gpu --num_processes 3 train_recipe_autoregressive_prompt.py
115
+ ```
116
+
117
+ See the [Accelerate docs](https://huggingface.co/docs/accelerate/basic_tutorials/launch).
118
+
119
+ ## Adding a callback
120
+ 👟 Supports callbacks to customize your runs. You can either set callbacks in your model implementations or give them
121
+ explicitly to the Trainer.
122
+
123
+ Please check `trainer.utils.callbacks` to see available callbacks.
124
+
125
+ Here is how you provide an explicit call back to a 👟Trainer object for weight reinitialization.
126
+
127
+ ```python
128
+ def my_callback(trainer):
129
+ print(" > My callback was called.")
130
+
131
+ trainer = Trainer(..., callbacks={"on_init_end": my_callback})
132
+ trainer.fit()
133
+ ```
134
+
135
+ ## Profiling example
136
+
137
+ - Create the torch profiler as you like and pass it to the trainer.
138
+ ```python
139
+ import torch
140
+ profiler = torch.profiler.profile(
141
+ activities=[
142
+ torch.profiler.ProfilerActivity.CPU,
143
+ torch.profiler.ProfilerActivity.CUDA,
144
+ ],
145
+ schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
146
+ on_trace_ready=torch.profiler.tensorboard_trace_handler("./profiler/"),
147
+ record_shapes=True,
148
+ profile_memory=True,
149
+ with_stack=True,
150
+ )
151
+ prof = trainer.profile_fit(profiler, epochs=1, small_run=64)
152
+ then run Tensorboard
153
+ ```
154
+ - Run the tensorboard.
155
+ ```console
156
+ tensorboard --logdir="./profiler/"
157
+ ```
158
+
159
+ ## Supported Experiment Loggers
160
+ - [Tensorboard](https://www.tensorflow.org/tensorboard) - actively maintained
161
+ - [ClearML](https://clear.ml/) - actively maintained
162
+ - [MLFlow](https://mlflow.org/)
163
+ - [Aim](https://aimstack.io/)
164
+ - [WandDB](https://wandb.ai/)
165
+
166
+ To add a new logger, you must subclass [BaseDashboardLogger](trainer/logging/base_dash_logger.py) and overload its functions.
@@ -0,0 +1,222 @@
1
+ Metadata-Version: 2.1
2
+ Name: coqui-tts-trainer
3
+ Version: 0.1.0
4
+ Summary: General purpose model trainer for PyTorch that is more flexible than it should be, by 🐸Coqui.
5
+ Home-page: https://github.com/idiap/coqui-ai-Trainer
6
+ Author: Eren Gölge
7
+ Author-email: egolge@coqui.ai
8
+ Maintainer: Enno Hermann
9
+ Maintainer-email: enno.hermann@gmail.com
10
+ License: Apache2
11
+ Project-URL: Documentation, https://github.com/idiap/coqui-ai-Trainer
12
+ Project-URL: Tracker, https://github.com/idiap/coqui-ai-Trainer/issues
13
+ Project-URL: Repository, https://github.com/idiap/coqui-ai-Trainer
14
+ Classifier: Environment :: Console
15
+ Classifier: Natural Language :: English
16
+ Classifier: Development Status :: 3 - Alpha
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: License :: OSI Approved :: Apache Software License
19
+ Classifier: Operating System :: OS Independent
20
+ Classifier: Programming Language :: Python :: 3.8
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Requires-Python: >=3.6.0, <3.13
26
+ Description-Content-Type: text/markdown
27
+ Requires-Dist: torch>=1.7
28
+ Requires-Dist: coqpit
29
+ Requires-Dist: psutil
30
+ Requires-Dist: fsspec
31
+ Requires-Dist: tensorboard
32
+ Requires-Dist: soundfile
33
+ Provides-Extra: dev
34
+ Requires-Dist: black; extra == "dev"
35
+ Requires-Dist: coverage; extra == "dev"
36
+ Requires-Dist: isort; extra == "dev"
37
+ Requires-Dist: pytest; extra == "dev"
38
+ Requires-Dist: pylint; extra == "dev"
39
+ Requires-Dist: accelerate; extra == "dev"
40
+ Provides-Extra: test
41
+ Requires-Dist: torchvision; extra == "test"
42
+ Provides-Extra: all
43
+ Requires-Dist: torch>=1.7; extra == "all"
44
+ Requires-Dist: coqpit; extra == "all"
45
+ Requires-Dist: psutil; extra == "all"
46
+ Requires-Dist: fsspec; extra == "all"
47
+ Requires-Dist: tensorboard; extra == "all"
48
+ Requires-Dist: soundfile; extra == "all"
49
+ Requires-Dist: black; extra == "all"
50
+ Requires-Dist: coverage; extra == "all"
51
+ Requires-Dist: isort; extra == "all"
52
+ Requires-Dist: pytest; extra == "all"
53
+ Requires-Dist: pylint; extra == "all"
54
+ Requires-Dist: accelerate; extra == "all"
55
+ Requires-Dist: torchvision; extra == "all"
56
+
57
+ <p align="center"><img src="https://user-images.githubusercontent.com/1402048/151947958-0bcadf38-3a82-4b4e-96b4-a38d3721d737.png" align="right" height="255px" /></p>
58
+
59
+ # 👟 Trainer
60
+ An opinionated general purpose model trainer on PyTorch with a simple code base.
61
+ Fork of the [original, unmaintained repository](https://github.com/coqui-ai/Trainer). New PyPI package: [coqui-tts-trainer](https://pypi.org/project/coqui-tts-trainer)
62
+
63
+ ## Installation
64
+
65
+ From Github:
66
+
67
+ ```console
68
+ git clone https://github.com/idiap/coqui-ai-Trainer
69
+ cd coqui-ai-Trainer
70
+ make install
71
+ ```
72
+
73
+ From PyPI:
74
+
75
+ ```console
76
+ pip install coqui-tts-trainer
77
+ ```
78
+
79
+ ## Implementing a model
80
+ Subclass and overload the functions in the [```TrainerModel()```](trainer/model.py)
81
+
82
+
83
+ ## Training a model with auto-optimization
84
+ See the [MNIST example](examples/train_mnist.py).
85
+
86
+
87
+ ## Training a model with advanced optimization
88
+ With 👟 you can define the whole optimization cycle as you want as the in GAN example below. It enables more
89
+ under-the-hood control and flexibility for more advanced training loops.
90
+
91
+ You just have to use the ```scaled_backward()``` function to handle mixed precision training.
92
+
93
+ ```python
94
+ ...
95
+
96
+ def optimize(self, batch, trainer):
97
+ imgs, _ = batch
98
+
99
+ # sample noise
100
+ z = torch.randn(imgs.shape[0], 100)
101
+ z = z.type_as(imgs)
102
+
103
+ # train discriminator
104
+ imgs_gen = self.generator(z)
105
+ logits = self.discriminator(imgs_gen.detach())
106
+ fake = torch.zeros(imgs.size(0), 1)
107
+ fake = fake.type_as(imgs)
108
+ loss_fake = trainer.criterion(logits, fake)
109
+
110
+ valid = torch.ones(imgs.size(0), 1)
111
+ valid = valid.type_as(imgs)
112
+ logits = self.discriminator(imgs)
113
+ loss_real = trainer.criterion(logits, valid)
114
+ loss_disc = (loss_real + loss_fake) / 2
115
+
116
+ # step dicriminator
117
+ _, _ = self.scaled_backward(loss_disc, None, trainer, trainer.optimizer[0])
118
+
119
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
120
+ trainer.optimizer[0].step()
121
+ trainer.optimizer[0].zero_grad()
122
+
123
+ # train generator
124
+ imgs_gen = self.generator(z)
125
+
126
+ valid = torch.ones(imgs.size(0), 1)
127
+ valid = valid.type_as(imgs)
128
+
129
+ logits = self.discriminator(imgs_gen)
130
+ loss_gen = trainer.criterion(logits, valid)
131
+
132
+ # step generator
133
+ _, _ = self.scaled_backward(loss_gen, None, trainer, trainer.optimizer[1])
134
+ if trainer.total_steps_done % trainer.grad_accum_steps == 0:
135
+ trainer.optimizer[1].step()
136
+ trainer.optimizer[1].zero_grad()
137
+ return {"model_outputs": logits}, {"loss_gen": loss_gen, "loss_disc": loss_disc}
138
+
139
+ ...
140
+ ```
141
+
142
+ See the [GAN training example](examples/train_simple_gan.py) with Gradient Accumulation
143
+
144
+
145
+ ## Training with Batch Size Finder
146
+ see the test script [here](tests/test_train_batch_size_finder.py) for training with batch size finder.
147
+
148
+
149
+ The batch size finder starts at a default BS(defaults to 2048 but can also be user defined) and searches for the largest batch size that can fit on your hardware. you should expect for it to run multiple trainings until it finds it. to use it instead of calling ```trainer.fit()``` youll call ```trainer.fit_with_largest_batch_size(starting_batch_size=2048)``` with ```starting_batch_size``` being the batch the size you want to start the search with. very useful if you are wanting to use as much gpu mem as possible.
150
+
151
+ ## Training with DDP
152
+
153
+ ```console
154
+ $ python -m trainer.distribute --script path/to/your/train.py --gpus "0,1"
155
+ ```
156
+
157
+ We don't use ```.spawn()``` to initiate multi-gpu training since it causes certain limitations.
158
+
159
+ - Everything must the pickable.
160
+ - ```.spawn()``` trains the model in subprocesses and the model in the main process is not updated.
161
+ - DataLoader with N processes gets really slow when the N is large.
162
+
163
+ ## Training with [Accelerate](https://huggingface.co/docs/accelerate/index)
164
+
165
+ Setting `use_accelerate` in `TrainingArgs` to `True` will enable training with Accelerate.
166
+
167
+ You can also use it for multi-gpu or distributed training.
168
+
169
+ ```console
170
+ CUDA_VISIBLE_DEVICES="0,1,2" accelerate launch --multi_gpu --num_processes 3 train_recipe_autoregressive_prompt.py
171
+ ```
172
+
173
+ See the [Accelerate docs](https://huggingface.co/docs/accelerate/basic_tutorials/launch).
174
+
175
+ ## Adding a callback
176
+ 👟 Supports callbacks to customize your runs. You can either set callbacks in your model implementations or give them
177
+ explicitly to the Trainer.
178
+
179
+ Please check `trainer.utils.callbacks` to see available callbacks.
180
+
181
+ Here is how you provide an explicit call back to a 👟Trainer object for weight reinitialization.
182
+
183
+ ```python
184
+ def my_callback(trainer):
185
+ print(" > My callback was called.")
186
+
187
+ trainer = Trainer(..., callbacks={"on_init_end": my_callback})
188
+ trainer.fit()
189
+ ```
190
+
191
+ ## Profiling example
192
+
193
+ - Create the torch profiler as you like and pass it to the trainer.
194
+ ```python
195
+ import torch
196
+ profiler = torch.profiler.profile(
197
+ activities=[
198
+ torch.profiler.ProfilerActivity.CPU,
199
+ torch.profiler.ProfilerActivity.CUDA,
200
+ ],
201
+ schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
202
+ on_trace_ready=torch.profiler.tensorboard_trace_handler("./profiler/"),
203
+ record_shapes=True,
204
+ profile_memory=True,
205
+ with_stack=True,
206
+ )
207
+ prof = trainer.profile_fit(profiler, epochs=1, small_run=64)
208
+ then run Tensorboard
209
+ ```
210
+ - Run the tensorboard.
211
+ ```console
212
+ tensorboard --logdir="./profiler/"
213
+ ```
214
+
215
+ ## Supported Experiment Loggers
216
+ - [Tensorboard](https://www.tensorflow.org/tensorboard) - actively maintained
217
+ - [ClearML](https://clear.ml/) - actively maintained
218
+ - [MLFlow](https://mlflow.org/)
219
+ - [Aim](https://aimstack.io/)
220
+ - [WandDB](https://wandb.ai/)
221
+
222
+ To add a new logger, you must subclass [BaseDashboardLogger](trainer/logging/base_dash_logger.py) and overload its functions.
@@ -0,0 +1,44 @@
1
+ MANIFEST.in
2
+ README.md
3
+ pyproject.toml
4
+ requirements.dev.txt
5
+ requirements.test.txt
6
+ requirements.txt
7
+ setup.cfg
8
+ setup.py
9
+ coqui_tts_trainer.egg-info/PKG-INFO
10
+ coqui_tts_trainer.egg-info/SOURCES.txt
11
+ coqui_tts_trainer.egg-info/dependency_links.txt
12
+ coqui_tts_trainer.egg-info/not-zip-safe
13
+ coqui_tts_trainer.egg-info/requires.txt
14
+ coqui_tts_trainer.egg-info/top_level.txt
15
+ tests/test_continue_train.py
16
+ tests/test_lr_schedulers.py
17
+ tests/test_num_gpus.py
18
+ tests/test_train_batch_size_finder.py
19
+ tests/test_train_gan.py
20
+ tests/test_train_mnist.py
21
+ trainer/VERSION
22
+ trainer/__init__.py
23
+ trainer/callbacks.py
24
+ trainer/distribute.py
25
+ trainer/generic_utils.py
26
+ trainer/io.py
27
+ trainer/logger.py
28
+ trainer/model.py
29
+ trainer/torch.py
30
+ trainer/trainer.py
31
+ trainer/trainer_utils.py
32
+ trainer/logging/__init__.py
33
+ trainer/logging/aim_logger.py
34
+ trainer/logging/base_dash_logger.py
35
+ trainer/logging/clearml_logger.py
36
+ trainer/logging/console_logger.py
37
+ trainer/logging/dummy_logger.py
38
+ trainer/logging/mlflow_logger.py
39
+ trainer/logging/tensorboard_logger.py
40
+ trainer/logging/wandb_logger.py
41
+ trainer/utils/__init__.py
42
+ trainer/utils/cpu_memory.py
43
+ trainer/utils/cuda_memory.py
44
+ trainer/utils/distributed.py