reax 0.6.14.dev1__tar.gz → 0.7.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/PKG-INFO +69 -69
- reax-0.7.0.dev2/README.rst +163 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/pyproject.toml +120 -28
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/__init__.py +1 -1
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/_engine.py +14 -9
- reax-0.7.0.dev2/src/reax/_module_hooks.py +313 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/__init__.py +6 -2
- reax-0.7.0.dev2/src/reax/data/_datasource_manager.py +179 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/_datasources.py +27 -13
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/_loaders.py +6 -4
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/collate.py +4 -2
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/datamodules.py +15 -11
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/datasets.py +4 -5
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/fetchers.py +33 -12
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/samplers.py +74 -102
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/demos/boring_classes.py +5 -5
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/demos/mnist.py +21 -20
- reax-0.7.0.dev2/src/reax/hooks.py +280 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/keys.py +4 -4
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/lightning/rank_zero.py +23 -4
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/early_stopping.py +6 -6
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/model_checkpoint.py +63 -36
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/progress/progress_bar.py +5 -6
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/progress/rich_progress.py +64 -49
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/progress/tqdm_progress.py +57 -32
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/csv_logs.py +7 -7
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/logger.py +5 -5
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/pandas.py +11 -10
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/tensorboard.py +2 -2
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/_classification.py +33 -38
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/_metric.py +50 -55
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/_registry.py +33 -19
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/aggregation.py +40 -48
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/collections.py +2 -2
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/jm/types.py +3 -3
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/jm/utils.py +3 -3
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/regression.py +30 -34
- reax-0.7.0.dev2/src/reax/metrics/utils.py +252 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/modules.py +28 -9
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/optimizers.py +19 -9
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/plugins.py +1 -1
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/results.py +4 -3
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/saving.py +6 -1
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/stages/__init__.py +4 -2
- reax-0.6.14.dev1/src/reax/stages/stages.py → reax-0.7.0.dev2/src/reax/stages/_loops.py +235 -274
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/stages/_timer.py +4 -5
- reax-0.7.0.dev2/src/reax/stages/common.py +263 -0
- reax-0.7.0.dev2/src/reax/stages/fit.py +478 -0
- reax-0.7.0.dev2/src/reax/stages/predict.py +148 -0
- reax-0.7.0.dev2/src/reax/stages/stages.py +349 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/stages/stats.py +47 -15
- reax-0.7.0.dev2/src/reax/stages/test.py +124 -0
- reax-0.7.0.dev2/src/reax/stages/train.py +455 -0
- reax-0.7.0.dev2/src/reax/stages/validation.py +153 -0
- reax-0.7.0.dev2/src/reax/strategies/__init__.py +14 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_jax.py +10 -10
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_parallel.py +9 -14
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_strategies.py +3 -5
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_utils.py +1 -1
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/testing.py +13 -8
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/training/_checkpointing.py +12 -7
- reax-0.7.0.dev2/src/reax/training/_hooks_map.py +129 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/training/_logger_connector.py +10 -5
- reax-0.7.0.dev2/src/reax/training/_utils.py +76 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/training/trainer.py +283 -255
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/types.py +4 -2
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/utils/containers.py +3 -3
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/utils/events.py +13 -5
- reax-0.6.14.dev1/README.rst +0 -166
- reax-0.6.14.dev1/src/reax/_module_hooks.py +0 -147
- reax-0.6.14.dev1/src/reax/data/_datasource_manager.py +0 -130
- reax-0.6.14.dev1/src/reax/hooks.py +0 -190
- reax-0.6.14.dev1/src/reax/metrics/utils.py +0 -230
- reax-0.6.14.dev1/src/reax/stages/common.py +0 -213
- reax-0.6.14.dev1/src/reax/stages/fit.py +0 -587
- reax-0.6.14.dev1/src/reax/stages/predict.py +0 -84
- reax-0.6.14.dev1/src/reax/stages/test.py +0 -69
- reax-0.6.14.dev1/src/reax/stages/train.py +0 -206
- reax-0.6.14.dev1/src/reax/stages/validation.py +0 -90
- reax-0.6.14.dev1/src/reax/strategies/__init__.py +0 -7
- reax-0.6.14.dev1/src/reax/training/_deprecated.py +0 -95
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/LICENSE.txt +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/_types.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/dataloaders.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/kfold.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/data/utils.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/demos/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/exceptions.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/lightning/LICENSE +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/lightning/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/checkpointer.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/progress/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/listeners/utils.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/_utils.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/loggers/mlflow.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/_evaluators.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/_globals.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/metrics/jm/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/profilers/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/profilers/jax_profiler.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/profilers/profiler.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/random.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_factory.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/strategies/_single_device.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/training/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/training/_checkpoint_connector.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/utils/__init__.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/utils/arrays.py +0 -0
- {reax-0.6.14.dev1 → reax-0.7.0.dev2}/src/reax/utils/grads.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: reax
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0.dev2
|
|
4
4
|
Summary: REAX: A simple training framework for JAX-based projects
|
|
5
5
|
Keywords: machine learning,jax,research
|
|
6
6
|
Author-email: Martin Uhrin <martin.uhrin.10@ucl.ac.uk>
|
|
@@ -30,17 +30,20 @@ Requires-Dist: stevedore
|
|
|
30
30
|
Requires-Dist: tensorboardX
|
|
31
31
|
Requires-Dist: tqdm
|
|
32
32
|
Requires-Dist: typing-extensions
|
|
33
|
-
Requires-Dist: black ; extra == "dev"
|
|
34
33
|
Requires-Dist: cloudpickle ; extra == "dev"
|
|
35
34
|
Requires-Dist: flit ; extra == "dev"
|
|
36
35
|
Requires-Dist: ipython ; extra == "dev"
|
|
37
36
|
Requires-Dist: mlflow<3.7 ; extra == "dev"
|
|
37
|
+
Requires-Dist: omegaconf ; extra == "dev"
|
|
38
38
|
Requires-Dist: pandas ; extra == "dev"
|
|
39
39
|
Requires-Dist: pytest ; extra == "dev"
|
|
40
40
|
Requires-Dist: pytest-cov ; extra == "dev"
|
|
41
41
|
Requires-Dist: pytest-env ; extra == "dev"
|
|
42
|
+
Requires-Dist: pytest-timeout ; extra == "dev"
|
|
42
43
|
Requires-Dist: pre-commit ; extra == "dev"
|
|
43
44
|
Requires-Dist: pylint ; extra == "dev"
|
|
45
|
+
Requires-Dist: ruff ; extra == "dev"
|
|
46
|
+
Requires-Dist: ty==0.0.84 ; extra == "dev"
|
|
44
47
|
Requires-Dist: docutils ; extra == "docs"
|
|
45
48
|
Requires-Dist: jupyter ; extra == "docs"
|
|
46
49
|
Requires-Dist: myst-nb ; extra == "docs"
|
|
@@ -83,80 +86,76 @@ REAX
|
|
|
83
86
|
.. image:: https://img.shields.io/pypi/l/reax.svg
|
|
84
87
|
:target: https://pypi.python.org/pypi/reax/
|
|
85
88
|
|
|
86
|
-
.. image:: https://img.shields.io/badge/
|
|
87
|
-
:target: https://github.com/
|
|
89
|
+
.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
|
|
90
|
+
:target: https://github.com/astral-sh/ruff
|
|
91
|
+
:alt: Ruff
|
|
88
92
|
|
|
89
93
|
|
|
90
|
-
|
|
94
|
+
**Scalable, flexible training for JAX, inspired by the simplicity of PyTorch Lightning.**
|
|
91
95
|
|
|
92
|
-
REAX
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
REAX is a minimal and high-performance framework for training JAX models, designed to simplify
|
|
96
|
-
research workflows. Inspired by PyTorch Lightning, it brings similar high-level abstractions and
|
|
97
|
-
scalability to JAX users, making it easier to scale models across multiple GPUs with minimal
|
|
98
|
-
boilerplate. 🚀
|
|
99
|
-
|
|
100
|
-
A Port of PyTorch Lightning to JAX
|
|
101
|
-
----------------------------------
|
|
102
|
-
|
|
103
|
-
Much of REAX is built by porting the best practices and abstractions of **PyTorch Lightning** to
|
|
104
|
-
the **JAX** ecosystem. If you're familiar with PyTorch Lightning, you'll recognize concepts like:
|
|
105
|
-
|
|
106
|
-
- Training loops ⚡
|
|
107
|
-
- Multi-GPU training 🖥️
|
|
108
|
-
- Logging and checkpointing 💾
|
|
109
|
-
|
|
110
|
-
However, REAX has been designed with JAX-specific optimizations, ensuring high performance without
|
|
111
|
-
sacrificing flexibility.
|
|
96
|
+
REAX is a lightweight framework for training JAX models. It takes care of the training loop,
|
|
97
|
+
logging, checkpointing and scaling across devices, so that your code can focus on the model and
|
|
98
|
+
the science. 🚀
|
|
112
99
|
|
|
113
100
|
Why REAX? 🌟
|
|
114
101
|
------------
|
|
115
102
|
|
|
116
|
-
- **
|
|
117
|
-
|
|
118
|
-
- **
|
|
119
|
-
|
|
103
|
+
- **Bring your own model** 🔧: REAX works with any JAX library, e.g. Flax Linen, Flax NNX or
|
|
104
|
+
Equinox. Your model stays plain JAX code.
|
|
105
|
+
- **Minimal boilerplate** 🧩: write a ``reax.Module`` with your training step and optimiser, and
|
|
106
|
+
let the ``reax.Trainer`` run fitting, validation, testing and prediction.
|
|
107
|
+
- **Batteries included** 💾: checkpointing, early stopping, progress bars, metrics, and loggers
|
|
108
|
+
for CSV, TensorBoard and MLflow.
|
|
109
|
+
- **Scales with JAX** ⚡: run on multiple devices and processes with minimal code changes.
|
|
110
|
+
- **Stay in control** 🎯: hook into every stage of training with module hooks and listeners, or
|
|
111
|
+
drop down to the ``reax.Engine`` to write your own training loop.
|
|
120
112
|
|
|
121
113
|
Installation 🛠️
|
|
122
114
|
---------------
|
|
123
115
|
|
|
124
|
-
|
|
116
|
+
With pip:
|
|
125
117
|
|
|
126
118
|
.. code-block:: shell
|
|
127
119
|
|
|
128
120
|
pip install reax
|
|
129
121
|
|
|
122
|
+
or with `uv <https://docs.astral.sh/uv/>`_, to add REAX to your project:
|
|
130
123
|
|
|
131
|
-
|
|
132
|
-
|
|
124
|
+
.. code-block:: shell
|
|
125
|
+
|
|
126
|
+
uv add reax
|
|
127
|
+
|
|
128
|
+
Example
|
|
129
|
+
-------
|
|
133
130
|
|
|
134
|
-
|
|
131
|
+
Train an autoencoder on MNIST, using a Flax Linen model:
|
|
135
132
|
|
|
136
133
|
.. code-block:: python
|
|
137
134
|
|
|
138
|
-
# main.py
|
|
139
135
|
from functools import partial
|
|
140
|
-
|
|
136
|
+
|
|
137
|
+
from flax import linen
|
|
138
|
+
import jax
|
|
139
|
+
import optax
|
|
140
|
+
|
|
141
|
+
import reax
|
|
141
142
|
from reax.demos import mnist
|
|
142
143
|
|
|
143
144
|
|
|
144
145
|
class Autoencoder(linen.Module):
|
|
145
146
|
def setup(self):
|
|
146
|
-
super().__init__()
|
|
147
147
|
self.encoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(3)])
|
|
148
148
|
self.decoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(28 * 28)])
|
|
149
149
|
|
|
150
150
|
def __call__(self, x):
|
|
151
|
-
|
|
152
|
-
return self.decoder(z)
|
|
151
|
+
return self.decoder(self.encoder(x))
|
|
153
152
|
|
|
154
153
|
|
|
155
154
|
# --------------------------------
|
|
156
155
|
# Step 1: Define a REAX Module
|
|
157
156
|
# --------------------------------
|
|
158
|
-
# A
|
|
159
|
-
#
|
|
157
|
+
# A reax.Module defines a full *system* (e.g. an LLM, diffusion model, autoencoder or
|
|
158
|
+
# image classifier): the model, how to train it and how to optimise it.
|
|
160
159
|
class ReaxAutoEncoder(reax.Module):
|
|
161
160
|
def __init__(self):
|
|
162
161
|
super().__init__()
|
|
@@ -164,64 +163,65 @@ Define the training workflow. Here's a toy example:
|
|
|
164
163
|
|
|
165
164
|
def configure_model(self, stage: reax.Stage, batch, /):
|
|
166
165
|
if self.parameters() is None:
|
|
167
|
-
#
|
|
166
|
+
# Initialise the parameters using an example batch
|
|
168
167
|
x, _ = batch
|
|
169
|
-
|
|
170
|
-
# Initialize Flax Linen model with RNGs and example input
|
|
171
|
-
params = self.ae.init(self.rngs(), x)
|
|
168
|
+
params = self.ae.init(self.rngs(), x.reshape(x.shape[0], -1))
|
|
172
169
|
self.set_parameters(params)
|
|
173
170
|
|
|
174
171
|
def __call__(self, x):
|
|
175
|
-
# Use the full autoencoder model for forward pass
|
|
176
172
|
return self.ae.apply(self.parameters(), x)
|
|
177
173
|
|
|
178
174
|
def training_step(self, batch, batch_idx):
|
|
179
175
|
x, _ = batch
|
|
180
176
|
x = x.reshape(x.shape[0], -1)
|
|
181
|
-
|
|
182
|
-
loss, grads = jax.value_and_grad(self.loss_fn, argnums=0)(
|
|
183
|
-
self.parameters(), x, self.ae.apply
|
|
184
|
-
)
|
|
177
|
+
loss, grads = jax.value_and_grad(self.loss_fn)(self.parameters(), x, self.ae.apply)
|
|
185
178
|
self.log("train_loss", loss, on_step=True, prog_bar=True)
|
|
179
|
+
# Return the gradients and REAX takes care of the optimiser update
|
|
186
180
|
return loss, grads
|
|
187
181
|
|
|
182
|
+
def validation_step(self, batch, batch_idx):
|
|
183
|
+
x, _ = batch
|
|
184
|
+
x = x.reshape(x.shape[0], -1)
|
|
185
|
+
self.log("val_loss", self.loss_fn(self.parameters(), x, self.ae.apply), prog_bar=True)
|
|
186
|
+
|
|
188
187
|
@staticmethod
|
|
189
188
|
@partial(jax.jit, static_argnums=2)
|
|
190
189
|
def loss_fn(params, x, apply_fn):
|
|
191
|
-
"""Compute reconstruction loss.
|
|
192
|
-
|
|
193
|
-
Static method for JIT compilation. Receives parameters and apply function.
|
|
194
|
-
"""
|
|
195
190
|
predictions = apply_fn(params, x)
|
|
196
191
|
return optax.losses.squared_error(predictions, x).mean()
|
|
197
192
|
|
|
198
193
|
def configure_optimizers(self):
|
|
199
194
|
opt = optax.adam(learning_rate=1e-3)
|
|
200
|
-
|
|
201
|
-
return opt, state
|
|
195
|
+
return opt, opt.init(self.parameters())
|
|
202
196
|
|
|
203
197
|
|
|
204
198
|
# -------------------
|
|
205
199
|
# Step 2: Define data
|
|
206
200
|
# -------------------
|
|
207
|
-
dataset = mnist.MnistDataset(download=True)
|
|
208
201
|
trainer = reax.Trainer()
|
|
209
|
-
|
|
202
|
+
dataset = mnist.MnistDataset(download=True)
|
|
203
|
+
train, val = reax.data.random_split(trainer.rngs, dataset, [55_000, 5_000])
|
|
210
204
|
|
|
211
205
|
# -------------------
|
|
212
206
|
# Step 3: Train
|
|
213
207
|
# -------------------
|
|
214
208
|
autoencoder = ReaxAutoEncoder()
|
|
215
|
-
trainer.fit(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
209
|
+
trainer.fit(
|
|
210
|
+
autoencoder,
|
|
211
|
+
reax.ReaxDataLoader(train, batch_size=64, shuffle=True),
|
|
212
|
+
reax.ReaxDataLoader(val, batch_size=64),
|
|
213
|
+
max_epochs=5,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
More examples, including Flax NNX and Equinox models, are in the
|
|
217
|
+
`documentation <https://reax.readthedocs.io/>`_.
|
|
218
|
+
|
|
219
|
+
Coming from PyTorch Lightning?
|
|
220
|
+
------------------------------
|
|
221
|
+
|
|
222
|
+
REAX ports the abstractions and best practices of PyTorch Lightning to JAX, and parts of its
|
|
223
|
+
code are adapted directly from Lightning. If you know Lightning, the ``Trainer``, ``Module``,
|
|
224
|
+
hooks, listeners (Lightning's callbacks) and loggers will feel familiar. Where it matters, REAX
|
|
225
|
+
departs from Lightning to fit JAX: parameters and optimiser state are explicit, and training steps
|
|
226
|
+
return gradients rather than relying on ``loss.backward()``.
|
|
227
227
|
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
REAX
|
|
2
|
+
====
|
|
3
|
+
|
|
4
|
+
.. image:: https://codecov.io/gh/muhrin/reax/branch/develop/graph/badge.svg
|
|
5
|
+
:target: https://codecov.io/gh/muhrin/reax
|
|
6
|
+
:alt: Coverage
|
|
7
|
+
|
|
8
|
+
.. image:: https://github.com/camml-lab/reax/actions/workflows/ci.yml/badge.svg
|
|
9
|
+
:target: https://github.com/camml-lab/reax/actions/workflows/ci.yml
|
|
10
|
+
:alt: Tests
|
|
11
|
+
|
|
12
|
+
.. image:: https://readthedocs.org/projects/reax/badge/
|
|
13
|
+
:target: https://reax.readthedocs.io/
|
|
14
|
+
:alt: Documentation
|
|
15
|
+
|
|
16
|
+
.. image:: https://img.shields.io/pypi/v/reax.svg
|
|
17
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
18
|
+
:alt: Latest Version
|
|
19
|
+
|
|
20
|
+
.. image:: https://img.shields.io/pypi/pyversions/reax.svg
|
|
21
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
22
|
+
|
|
23
|
+
.. image:: https://img.shields.io/pypi/l/reax.svg
|
|
24
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
25
|
+
|
|
26
|
+
.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
|
|
27
|
+
:target: https://github.com/astral-sh/ruff
|
|
28
|
+
:alt: Ruff
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
**Scalable, flexible training for JAX, inspired by the simplicity of PyTorch Lightning.**
|
|
32
|
+
|
|
33
|
+
REAX is a lightweight framework for training JAX models. It takes care of the training loop,
|
|
34
|
+
logging, checkpointing and scaling across devices, so that your code can focus on the model and
|
|
35
|
+
the science. 🚀
|
|
36
|
+
|
|
37
|
+
Why REAX? 🌟
|
|
38
|
+
------------
|
|
39
|
+
|
|
40
|
+
- **Bring your own model** 🔧: REAX works with any JAX library, e.g. Flax Linen, Flax NNX or
|
|
41
|
+
Equinox. Your model stays plain JAX code.
|
|
42
|
+
- **Minimal boilerplate** 🧩: write a ``reax.Module`` with your training step and optimiser, and
|
|
43
|
+
let the ``reax.Trainer`` run fitting, validation, testing and prediction.
|
|
44
|
+
- **Batteries included** 💾: checkpointing, early stopping, progress bars, metrics, and loggers
|
|
45
|
+
for CSV, TensorBoard and MLflow.
|
|
46
|
+
- **Scales with JAX** ⚡: run on multiple devices and processes with minimal code changes.
|
|
47
|
+
- **Stay in control** 🎯: hook into every stage of training with module hooks and listeners, or
|
|
48
|
+
drop down to the ``reax.Engine`` to write your own training loop.
|
|
49
|
+
|
|
50
|
+
Installation 🛠️
|
|
51
|
+
---------------
|
|
52
|
+
|
|
53
|
+
With pip:
|
|
54
|
+
|
|
55
|
+
.. code-block:: shell
|
|
56
|
+
|
|
57
|
+
pip install reax
|
|
58
|
+
|
|
59
|
+
or with `uv <https://docs.astral.sh/uv/>`_, to add REAX to your project:
|
|
60
|
+
|
|
61
|
+
.. code-block:: shell
|
|
62
|
+
|
|
63
|
+
uv add reax
|
|
64
|
+
|
|
65
|
+
Example
|
|
66
|
+
-------
|
|
67
|
+
|
|
68
|
+
Train an autoencoder on MNIST, using a Flax Linen model:
|
|
69
|
+
|
|
70
|
+
.. code-block:: python
|
|
71
|
+
|
|
72
|
+
from functools import partial
|
|
73
|
+
|
|
74
|
+
from flax import linen
|
|
75
|
+
import jax
|
|
76
|
+
import optax
|
|
77
|
+
|
|
78
|
+
import reax
|
|
79
|
+
from reax.demos import mnist
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class Autoencoder(linen.Module):
|
|
83
|
+
def setup(self):
|
|
84
|
+
self.encoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(3)])
|
|
85
|
+
self.decoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(28 * 28)])
|
|
86
|
+
|
|
87
|
+
def __call__(self, x):
|
|
88
|
+
return self.decoder(self.encoder(x))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# --------------------------------
|
|
92
|
+
# Step 1: Define a REAX Module
|
|
93
|
+
# --------------------------------
|
|
94
|
+
# A reax.Module defines a full *system* (e.g. an LLM, diffusion model, autoencoder or
|
|
95
|
+
# image classifier): the model, how to train it and how to optimise it.
|
|
96
|
+
class ReaxAutoEncoder(reax.Module):
|
|
97
|
+
def __init__(self):
|
|
98
|
+
super().__init__()
|
|
99
|
+
self.ae = Autoencoder()
|
|
100
|
+
|
|
101
|
+
def configure_model(self, stage: reax.Stage, batch, /):
|
|
102
|
+
if self.parameters() is None:
|
|
103
|
+
# Initialise the parameters using an example batch
|
|
104
|
+
x, _ = batch
|
|
105
|
+
params = self.ae.init(self.rngs(), x.reshape(x.shape[0], -1))
|
|
106
|
+
self.set_parameters(params)
|
|
107
|
+
|
|
108
|
+
def __call__(self, x):
|
|
109
|
+
return self.ae.apply(self.parameters(), x)
|
|
110
|
+
|
|
111
|
+
def training_step(self, batch, batch_idx):
|
|
112
|
+
x, _ = batch
|
|
113
|
+
x = x.reshape(x.shape[0], -1)
|
|
114
|
+
loss, grads = jax.value_and_grad(self.loss_fn)(self.parameters(), x, self.ae.apply)
|
|
115
|
+
self.log("train_loss", loss, on_step=True, prog_bar=True)
|
|
116
|
+
# Return the gradients and REAX takes care of the optimiser update
|
|
117
|
+
return loss, grads
|
|
118
|
+
|
|
119
|
+
def validation_step(self, batch, batch_idx):
|
|
120
|
+
x, _ = batch
|
|
121
|
+
x = x.reshape(x.shape[0], -1)
|
|
122
|
+
self.log("val_loss", self.loss_fn(self.parameters(), x, self.ae.apply), prog_bar=True)
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
@partial(jax.jit, static_argnums=2)
|
|
126
|
+
def loss_fn(params, x, apply_fn):
|
|
127
|
+
predictions = apply_fn(params, x)
|
|
128
|
+
return optax.losses.squared_error(predictions, x).mean()
|
|
129
|
+
|
|
130
|
+
def configure_optimizers(self):
|
|
131
|
+
opt = optax.adam(learning_rate=1e-3)
|
|
132
|
+
return opt, opt.init(self.parameters())
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# -------------------
|
|
136
|
+
# Step 2: Define data
|
|
137
|
+
# -------------------
|
|
138
|
+
trainer = reax.Trainer()
|
|
139
|
+
dataset = mnist.MnistDataset(download=True)
|
|
140
|
+
train, val = reax.data.random_split(trainer.rngs, dataset, [55_000, 5_000])
|
|
141
|
+
|
|
142
|
+
# -------------------
|
|
143
|
+
# Step 3: Train
|
|
144
|
+
# -------------------
|
|
145
|
+
autoencoder = ReaxAutoEncoder()
|
|
146
|
+
trainer.fit(
|
|
147
|
+
autoencoder,
|
|
148
|
+
reax.ReaxDataLoader(train, batch_size=64, shuffle=True),
|
|
149
|
+
reax.ReaxDataLoader(val, batch_size=64),
|
|
150
|
+
max_epochs=5,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
More examples, including Flax NNX and Equinox models, are in the
|
|
154
|
+
`documentation <https://reax.readthedocs.io/>`_.
|
|
155
|
+
|
|
156
|
+
Coming from PyTorch Lightning?
|
|
157
|
+
------------------------------
|
|
158
|
+
|
|
159
|
+
REAX ports the abstractions and best practices of PyTorch Lightning to JAX, and parts of its
|
|
160
|
+
code are adapted directly from Lightning. If you know Lightning, the ``Trainer``, ``Module``,
|
|
161
|
+
hooks, listeners (Lightning's callbacks) and loggers will feel familiar. Where it matters, REAX
|
|
162
|
+
departs from Lightning to fit JAX: parameters and optimiser state are explicit, and training steps
|
|
163
|
+
return gradients rather than relying on ``loss.backward()``.
|
|
@@ -56,17 +56,20 @@ docs = [
|
|
|
56
56
|
'sphinx-autobuild',
|
|
57
57
|
]
|
|
58
58
|
dev = [
|
|
59
|
-
'black',
|
|
60
59
|
'cloudpickle',
|
|
61
60
|
'flit',
|
|
62
61
|
'ipython',
|
|
63
62
|
"mlflow<3.7",
|
|
63
|
+
"omegaconf",
|
|
64
64
|
'pandas',
|
|
65
65
|
'pytest',
|
|
66
66
|
'pytest-cov',
|
|
67
67
|
'pytest-env',
|
|
68
|
+
'pytest-timeout',
|
|
68
69
|
'pre-commit',
|
|
69
70
|
'pylint',
|
|
71
|
+
'ruff',
|
|
72
|
+
'ty==0.0.84',
|
|
70
73
|
]
|
|
71
74
|
examples = [
|
|
72
75
|
"lightning",
|
|
@@ -96,22 +99,123 @@ mse = "reax.metrics:MeanSquaredError"
|
|
|
96
99
|
rmse = "reax.metrics:RootMeanSquareError"
|
|
97
100
|
mae = "reax.metrics:MeanAbsoluteError"
|
|
98
101
|
|
|
99
|
-
[tool.
|
|
102
|
+
[tool.ruff]
|
|
100
103
|
line-length = 100
|
|
101
|
-
|
|
104
|
+
target-version = "py310"
|
|
105
|
+
exclude = [
|
|
106
|
+
'migrations',
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
[tool.ruff.lint]
|
|
110
|
+
select = [
|
|
111
|
+
'B',
|
|
112
|
+
'B9',
|
|
113
|
+
'C90',
|
|
114
|
+
'E',
|
|
115
|
+
'F',
|
|
116
|
+
'I',
|
|
117
|
+
'S',
|
|
118
|
+
'UP',
|
|
119
|
+
'W',
|
|
120
|
+
]
|
|
121
|
+
ignore = [
|
|
122
|
+
'E203',
|
|
123
|
+
'E266',
|
|
124
|
+
'F403',
|
|
125
|
+
'F722',
|
|
126
|
+
'F821',
|
|
127
|
+
'S101',
|
|
128
|
+
'S603',
|
|
129
|
+
'S607',
|
|
130
|
+
]
|
|
102
131
|
|
|
103
|
-
[tool.isort]
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
132
|
+
[tool.ruff.lint.isort]
|
|
133
|
+
force-sort-within-sections = true
|
|
134
|
+
|
|
135
|
+
[tool.ruff.lint.mccabe]
|
|
136
|
+
max-complexity = 18
|
|
137
|
+
|
|
138
|
+
[tool.ruff.lint.per-file-ignores]
|
|
139
|
+
'src/reax/lightning/*.py' = [
|
|
140
|
+
'C90',
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
[tool.ty.src]
|
|
144
|
+
# TODO: These files have outstanding ty errors, remove them from this list as they are fixed
|
|
145
|
+
exclude = [
|
|
146
|
+
'docs/source/get_started/introduction.ipynb',
|
|
147
|
+
'docs/source/get_started/quick_start.ipynb',
|
|
148
|
+
'examples/autoencoder.py',
|
|
149
|
+
'src/reax/data/_datasource_manager.py',
|
|
150
|
+
'src/reax/data/_loaders.py',
|
|
151
|
+
'src/reax/data/_types.py',
|
|
152
|
+
'src/reax/data/collate.py',
|
|
153
|
+
'src/reax/data/datamodules.py',
|
|
154
|
+
'src/reax/data/datasets.py',
|
|
155
|
+
'src/reax/data/fetchers.py',
|
|
156
|
+
'src/reax/data/samplers.py',
|
|
157
|
+
'src/reax/data/utils.py',
|
|
158
|
+
'src/reax/demos/boring_classes.py',
|
|
159
|
+
'src/reax/demos/mnist.py',
|
|
160
|
+
'src/reax/listeners/early_stopping.py',
|
|
161
|
+
'src/reax/listeners/model_checkpoint.py',
|
|
162
|
+
'src/reax/listeners/progress/tqdm_progress.py',
|
|
163
|
+
'src/reax/loggers/_utils.py',
|
|
164
|
+
'src/reax/loggers/csv_logs.py',
|
|
165
|
+
'src/reax/loggers/logger.py',
|
|
166
|
+
'src/reax/loggers/mlflow.py',
|
|
167
|
+
'src/reax/loggers/pandas.py',
|
|
168
|
+
'src/reax/loggers/tensorboard.py',
|
|
169
|
+
'src/reax/modules.py',
|
|
170
|
+
'src/reax/optimizers.py',
|
|
171
|
+
'src/reax/random.py',
|
|
172
|
+
'src/reax/results.py',
|
|
173
|
+
'src/reax/strategies/_factory.py',
|
|
174
|
+
'src/reax/strategies/_jax.py',
|
|
175
|
+
'src/reax/strategies/_parallel.py',
|
|
176
|
+
'src/reax/training/trainer.py',
|
|
177
|
+
'src/reax/utils/arrays.py',
|
|
178
|
+
'src/reax/utils/containers.py',
|
|
179
|
+
'src/reax/utils/events.py',
|
|
180
|
+
'tests/checkpointing/test_checkpoint_callback_frequency.py',
|
|
181
|
+
'tests/core/test_module.py',
|
|
182
|
+
'tests/core/todo_test_module.py',
|
|
183
|
+
'tests/data/test_collate.py',
|
|
184
|
+
'tests/data/test_datamodules_and_manager.py',
|
|
185
|
+
'tests/data/test_datasets.py',
|
|
186
|
+
'tests/data/test_fetchers.py',
|
|
187
|
+
'tests/data/test_kfold.py',
|
|
188
|
+
'tests/data/test_kfold_distributed.py',
|
|
189
|
+
'tests/data/test_loaders.py',
|
|
190
|
+
'tests/data/test_samplers.py',
|
|
191
|
+
'tests/data/test_utils.py',
|
|
192
|
+
'tests/engine/test_engine_ddp.py',
|
|
193
|
+
'tests/helpers/simple_models.py',
|
|
194
|
+
'tests/listeners/progress/test_rich_progress_bar.py',
|
|
195
|
+
'tests/listeners/test_early_stopping.py',
|
|
196
|
+
'tests/listeners/test_listeners.py',
|
|
197
|
+
'tests/listeners/test_model_checkpoint.py',
|
|
198
|
+
'tests/loggers/conftest.py',
|
|
199
|
+
'tests/loggers/test_mlflow.py',
|
|
200
|
+
'tests/loggers/test_pandas.py',
|
|
201
|
+
'tests/loggers/test_tensorboard.py',
|
|
202
|
+
'tests/models/test_hooks.py',
|
|
203
|
+
'tests/profilers/test_jax_profiler.py',
|
|
204
|
+
'tests/stages/test_timer.py',
|
|
205
|
+
'tests/strategies/test_parallel_strategy.py',
|
|
206
|
+
'tests/test_metrics.py',
|
|
207
|
+
'tests/training/test_dataloaders.py',
|
|
208
|
+
'tests/training/test_trainer.py',
|
|
209
|
+
'tests/utils/test_arrays.py',
|
|
210
|
+
'tests/utils/test_containers.py',
|
|
211
|
+
]
|
|
109
212
|
|
|
110
213
|
[tool.pylint.MASTER]
|
|
111
214
|
ignore-paths = '^src/reax/lightning/.*$' # These are copy-paste from lightning, so don't lint
|
|
112
215
|
|
|
113
216
|
[tool.pylint.format]
|
|
114
217
|
max-line-length = 100
|
|
218
|
+
max-module-lines = 1200
|
|
115
219
|
|
|
116
220
|
[tool.pylint.messages_control]
|
|
117
221
|
disable = [
|
|
@@ -136,15 +240,6 @@ max-returns = 8
|
|
|
136
240
|
max-positional-arguments = 10
|
|
137
241
|
max-public-methods = 22
|
|
138
242
|
|
|
139
|
-
[tool.bandit]
|
|
140
|
-
skips = [
|
|
141
|
-
"B101",
|
|
142
|
-
"B404" # This warns on import of subprocess
|
|
143
|
-
]
|
|
144
|
-
|
|
145
|
-
[tool.black]
|
|
146
|
-
line-length = 100
|
|
147
|
-
|
|
148
243
|
[tool.doc8]
|
|
149
244
|
max-line-length = 100
|
|
150
245
|
|
|
@@ -158,26 +253,23 @@ log_cli_level = "DEBUG"
|
|
|
158
253
|
|
|
159
254
|
[tool.pytest_env]
|
|
160
255
|
XLA_PYTHON_CLIENT_PREALLOCATE = false # Don't allow JAX to preallocate memory
|
|
256
|
+
JAX_PLATFORMS = "cpu" # Force tests to run on the CPU
|
|
161
257
|
|
|
162
258
|
[tool.pytest.ini_options]
|
|
163
259
|
minversion = '6.0'
|
|
164
260
|
testpaths = [
|
|
165
|
-
'
|
|
261
|
+
'tests',
|
|
166
262
|
]
|
|
167
263
|
filterwarnings = [
|
|
168
264
|
'ignore::DeprecationWarning:frozendict:',
|
|
169
265
|
]
|
|
266
|
+
markers = [
|
|
267
|
+
'multiproc: tests that spawn multiple JAX DDP subprocesses',
|
|
268
|
+
]
|
|
269
|
+
timeout = 600
|
|
270
|
+
timeout_method = 'thread'
|
|
170
271
|
#addopts = "--jaxtyping-packages=reax,beartype.beartype"
|
|
171
272
|
|
|
172
|
-
[tool.yapf]
|
|
173
|
-
align_closing_bracket_with_visual_indent = true
|
|
174
|
-
based_on_style = 'google'
|
|
175
|
-
coalesce_brackets = true
|
|
176
|
-
column_limit = 100
|
|
177
|
-
dedent_closing_brackets = true
|
|
178
|
-
indent_dictionary_value = false
|
|
179
|
-
split_arguments_when_comma_terminated = true
|
|
180
|
-
|
|
181
273
|
[tool.tox]
|
|
182
274
|
legacy_tox_ini = """
|
|
183
275
|
[tox]
|
|
@@ -12,11 +12,11 @@ import jax
|
|
|
12
12
|
import jaxtyping as jt
|
|
13
13
|
|
|
14
14
|
from . import data as data_
|
|
15
|
-
from . import hooks
|
|
15
|
+
from . import hooks, modules, optimizers, profilers, strategies
|
|
16
16
|
from . import loggers as loggers_
|
|
17
17
|
from . import metrics as metrics_
|
|
18
|
-
from . import modules, optimizers, profilers, strategies
|
|
19
18
|
from .data import DeviceDataLoader
|
|
19
|
+
from .lightning import rank_zero
|
|
20
20
|
from .training import _logger_connector
|
|
21
21
|
from .utils import events
|
|
22
22
|
|
|
@@ -61,10 +61,10 @@ class Engine:
|
|
|
61
61
|
logger: "reax.Logger | Iterable[reax.Logger] | bool | None" = True,
|
|
62
62
|
listeners: "list[reax.TrainerListener] | reax.TrainerListener | None" = None,
|
|
63
63
|
deterministic: bool = False,
|
|
64
|
-
rngs: nnx.Rngs = None,
|
|
64
|
+
rngs: nnx.Rngs | None = None,
|
|
65
65
|
default_root_dir: "reax.types.Path | None" = None,
|
|
66
66
|
profiler: "reax.Profiler | str | None" = None,
|
|
67
|
-
metric_evaluator: "reax.MetricEvaluator | None" = None,
|
|
67
|
+
metric_evaluator: "reax.metrics.MetricEvaluator | None" = None,
|
|
68
68
|
):
|
|
69
69
|
"""
|
|
70
70
|
Initializes the Engine with execution parameters and components.
|
|
@@ -106,6 +106,10 @@ class Engine:
|
|
|
106
106
|
self._strategy = strategy
|
|
107
107
|
else:
|
|
108
108
|
self._strategy = strategies.create(strategy, accelerator, devices=devices)
|
|
109
|
+
# The strategy knows this process's rank whatever launched it (e.g. REAX itself, MPI or
|
|
110
|
+
# SLURM), so make the rank-zero utilities agree with it. This has to happen before the
|
|
111
|
+
# loggers are created, as they only write from rank zero.
|
|
112
|
+
rank_zero.set_rank(self.process_index)
|
|
109
113
|
self._default_root_dir = (
|
|
110
114
|
os.fspath(default_root_dir) if default_root_dir is not None else os.getcwd()
|
|
111
115
|
)
|
|
@@ -133,9 +137,10 @@ class Engine:
|
|
|
133
137
|
After this is called, this object should no longer be interacted with,
|
|
134
138
|
as internal components (like event listeners and loggers) are cleared.
|
|
135
139
|
"""
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
self.
|
|
140
|
+
# Note: avoid referencing module globals here, as this can be called from `Trainer.__del__`
|
|
141
|
+
# during interpreter shutdown, when they may already have been cleared
|
|
142
|
+
self._events.reset()
|
|
143
|
+
self._loggers = []
|
|
139
144
|
|
|
140
145
|
@property
|
|
141
146
|
def default_root_dir(self) -> str:
|
|
@@ -467,7 +472,7 @@ def _(arg: data_.DataLoader, engine: Engine) -> data_.DataLoader:
|
|
|
467
472
|
def _init_loggers(
|
|
468
473
|
logger: "reax.Logger | Iterable[reax.Logger] | bool | None",
|
|
469
474
|
default_root_dir: str,
|
|
470
|
-
) -> list[
|
|
475
|
+
) -> "list[reax.Logger]":
|
|
471
476
|
"""Init loggers."""
|
|
472
477
|
if isinstance(logger, loggers_.Logger):
|
|
473
478
|
return [logger]
|
|
@@ -482,7 +487,7 @@ def _init_loggers(
|
|
|
482
487
|
|
|
483
488
|
|
|
484
489
|
def _init_profiler(profiler: "reax.Profiler | str | None" = None) -> "reax.Profiler":
|
|
485
|
-
profs: Final[dict[str, type
|
|
490
|
+
profs: Final[dict[str, type[profilers.Profiler]]] = {
|
|
486
491
|
"dummy": profilers.DummyProfiler,
|
|
487
492
|
"jax": profilers.JaxProfiler,
|
|
488
493
|
}
|