reax 0.7.0.dev1__tar.gz → 0.7.0.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/PKG-INFO +67 -68
- reax-0.7.0.dev3/README.rst +163 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/pyproject.toml +75 -2
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/__init__.py +1 -1
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/_engine.py +12 -6
- reax-0.7.0.dev3/src/reax/_module_hooks.py +313 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/_datasource_manager.py +0 -1
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/_datasources.py +7 -7
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/_loaders.py +48 -26
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/_types.py +7 -3
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/datasets.py +1 -3
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/samplers.py +75 -65
- reax-0.7.0.dev3/src/reax/hooks.py +280 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/keys.py +4 -4
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/lightning/rank_zero.py +23 -4
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/early_stopping.py +4 -4
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/model_checkpoint.py +46 -29
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/progress/progress_bar.py +3 -3
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/progress/rich_progress.py +63 -48
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/progress/tqdm_progress.py +56 -31
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/csv_logs.py +5 -5
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/logger.py +1 -2
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/pandas.py +9 -8
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/_metric.py +48 -53
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/_registry.py +31 -17
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/aggregation.py +32 -46
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/collections.py +1 -1
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/jm/utils.py +2 -2
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/regression.py +29 -33
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/utils.py +53 -27
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/modules.py +11 -8
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/optimizers.py +19 -9
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/results.py +4 -3
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/saving.py +4 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/stages/__init__.py +4 -2
- reax-0.7.0.dev1/src/reax/stages/stages.py → reax-0.7.0.dev3/src/reax/stages/_loops.py +234 -281
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/stages/_timer.py +2 -3
- reax-0.7.0.dev3/src/reax/stages/common.py +263 -0
- reax-0.7.0.dev3/src/reax/stages/fit.py +478 -0
- reax-0.7.0.dev3/src/reax/stages/predict.py +148 -0
- reax-0.7.0.dev3/src/reax/stages/stages.py +349 -0
- reax-0.7.0.dev3/src/reax/stages/stats.py +116 -0
- reax-0.7.0.dev3/src/reax/stages/test.py +124 -0
- reax-0.7.0.dev3/src/reax/stages/train.py +455 -0
- reax-0.7.0.dev3/src/reax/stages/validation.py +153 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/__init__.py +7 -1
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_parallel.py +4 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/testing.py +13 -6
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/training/_checkpointing.py +12 -7
- reax-0.7.0.dev3/src/reax/training/_hooks_map.py +129 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/training/_logger_connector.py +10 -5
- reax-0.7.0.dev3/src/reax/training/_utils.py +76 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/training/trainer.py +267 -234
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/types.py +4 -2
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/utils/events.py +8 -4
- reax-0.7.0.dev1/README.rst +0 -166
- reax-0.7.0.dev1/src/reax/_module_hooks.py +0 -147
- reax-0.7.0.dev1/src/reax/hooks.py +0 -190
- reax-0.7.0.dev1/src/reax/stages/common.py +0 -204
- reax-0.7.0.dev1/src/reax/stages/fit.py +0 -589
- reax-0.7.0.dev1/src/reax/stages/predict.py +0 -84
- reax-0.7.0.dev1/src/reax/stages/stats.py +0 -73
- reax-0.7.0.dev1/src/reax/stages/test.py +0 -69
- reax-0.7.0.dev1/src/reax/stages/train.py +0 -206
- reax-0.7.0.dev1/src/reax/stages/validation.py +0 -90
- reax-0.7.0.dev1/src/reax/training/_deprecated.py +0 -95
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/LICENSE.txt +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/collate.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/dataloaders.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/datamodules.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/fetchers.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/kfold.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/data/utils.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/demos/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/demos/boring_classes.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/demos/mnist.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/exceptions.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/lightning/LICENSE +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/lightning/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/checkpointer.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/progress/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/listeners/utils.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/_utils.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/mlflow.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/loggers/tensorboard.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/_classification.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/_evaluators.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/_globals.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/jm/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/metrics/jm/types.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/plugins.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/profilers/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/profilers/jax_profiler.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/profilers/profiler.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/random.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_factory.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_jax.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_single_device.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_strategies.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/strategies/_utils.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/training/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/training/_checkpoint_connector.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/utils/__init__.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/utils/arrays.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/utils/containers.py +0 -0
- {reax-0.7.0.dev1 → reax-0.7.0.dev3}/src/reax/utils/grads.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: reax
|
|
3
|
-
Version: 0.7.0.
|
|
3
|
+
Version: 0.7.0.dev3
|
|
4
4
|
Summary: REAX: A simple training framework for JAX-based projects
|
|
5
5
|
Keywords: machine learning,jax,research
|
|
6
6
|
Author-email: Martin Uhrin <martin.uhrin.10@ucl.ac.uk>
|
|
@@ -34,6 +34,7 @@ Requires-Dist: cloudpickle ; extra == "dev"
|
|
|
34
34
|
Requires-Dist: flit ; extra == "dev"
|
|
35
35
|
Requires-Dist: ipython ; extra == "dev"
|
|
36
36
|
Requires-Dist: mlflow<3.7 ; extra == "dev"
|
|
37
|
+
Requires-Dist: omegaconf ; extra == "dev"
|
|
37
38
|
Requires-Dist: pandas ; extra == "dev"
|
|
38
39
|
Requires-Dist: pytest ; extra == "dev"
|
|
39
40
|
Requires-Dist: pytest-cov ; extra == "dev"
|
|
@@ -42,6 +43,7 @@ Requires-Dist: pytest-timeout ; extra == "dev"
|
|
|
42
43
|
Requires-Dist: pre-commit ; extra == "dev"
|
|
43
44
|
Requires-Dist: pylint ; extra == "dev"
|
|
44
45
|
Requires-Dist: ruff ; extra == "dev"
|
|
46
|
+
Requires-Dist: ty==0.0.84 ; extra == "dev"
|
|
45
47
|
Requires-Dist: docutils ; extra == "docs"
|
|
46
48
|
Requires-Dist: jupyter ; extra == "docs"
|
|
47
49
|
Requires-Dist: myst-nb ; extra == "docs"
|
|
@@ -84,80 +86,76 @@ REAX
|
|
|
84
86
|
.. image:: https://img.shields.io/pypi/l/reax.svg
|
|
85
87
|
:target: https://pypi.python.org/pypi/reax/
|
|
86
88
|
|
|
87
|
-
.. image:: https://img.shields.io/badge/
|
|
88
|
-
:target: https://github.com/
|
|
89
|
+
.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
|
|
90
|
+
:target: https://github.com/astral-sh/ruff
|
|
91
|
+
:alt: Ruff
|
|
89
92
|
|
|
90
93
|
|
|
91
|
-
|
|
94
|
+
**Scalable, flexible training for JAX, inspired by the simplicity of PyTorch Lightning.**
|
|
92
95
|
|
|
93
|
-
REAX
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
REAX is a minimal and high-performance framework for training JAX models, designed to simplify
|
|
97
|
-
research workflows. Inspired by PyTorch Lightning, it brings similar high-level abstractions and
|
|
98
|
-
scalability to JAX users, making it easier to scale models across multiple GPUs with minimal
|
|
99
|
-
boilerplate. 🚀
|
|
100
|
-
|
|
101
|
-
A Port of PyTorch Lightning to JAX
|
|
102
|
-
----------------------------------
|
|
103
|
-
|
|
104
|
-
Much of REAX is built by porting the best practices and abstractions of **PyTorch Lightning** to
|
|
105
|
-
the **JAX** ecosystem. If you're familiar with PyTorch Lightning, you'll recognize concepts like:
|
|
106
|
-
|
|
107
|
-
- Training loops ⚡
|
|
108
|
-
- Multi-GPU training 🖥️
|
|
109
|
-
- Logging and checkpointing 💾
|
|
110
|
-
|
|
111
|
-
However, REAX has been designed with JAX-specific optimizations, ensuring high performance without
|
|
112
|
-
sacrificing flexibility.
|
|
96
|
+
REAX is a lightweight framework for training JAX models. It takes care of the training loop,
|
|
97
|
+
logging, checkpointing and scaling across devices, so that your code can focus on the model and
|
|
98
|
+
the science. 🚀
|
|
113
99
|
|
|
114
100
|
Why REAX? 🌟
|
|
115
101
|
------------
|
|
116
102
|
|
|
117
|
-
- **
|
|
118
|
-
|
|
119
|
-
- **
|
|
120
|
-
|
|
103
|
+
- **Bring your own model** 🔧: REAX works with any JAX library, e.g. Flax Linen, Flax NNX or
|
|
104
|
+
Equinox. Your model stays plain JAX code.
|
|
105
|
+
- **Minimal boilerplate** 🧩: write a ``reax.Module`` with your training step and optimiser, and
|
|
106
|
+
let the ``reax.Trainer`` run fitting, validation, testing and prediction.
|
|
107
|
+
- **Batteries included** 💾: checkpointing, early stopping, progress bars, metrics, and loggers
|
|
108
|
+
for CSV, TensorBoard and MLflow.
|
|
109
|
+
- **Scales with JAX** ⚡: run on multiple devices and processes with minimal code changes.
|
|
110
|
+
- **Stay in control** 🎯: hook into every stage of training with module hooks and listeners, or
|
|
111
|
+
drop down to the ``reax.Engine`` to write your own training loop.
|
|
121
112
|
|
|
122
113
|
Installation 🛠️
|
|
123
114
|
---------------
|
|
124
115
|
|
|
125
|
-
|
|
116
|
+
With pip:
|
|
126
117
|
|
|
127
118
|
.. code-block:: shell
|
|
128
119
|
|
|
129
120
|
pip install reax
|
|
130
121
|
|
|
122
|
+
or with `uv <https://docs.astral.sh/uv/>`_, to add REAX to your project:
|
|
131
123
|
|
|
132
|
-
|
|
133
|
-
|
|
124
|
+
.. code-block:: shell
|
|
125
|
+
|
|
126
|
+
uv add reax
|
|
127
|
+
|
|
128
|
+
Example
|
|
129
|
+
-------
|
|
134
130
|
|
|
135
|
-
|
|
131
|
+
Train an autoencoder on MNIST, using a Flax Linen model:
|
|
136
132
|
|
|
137
133
|
.. code-block:: python
|
|
138
134
|
|
|
139
|
-
# main.py
|
|
140
135
|
from functools import partial
|
|
141
|
-
|
|
136
|
+
|
|
137
|
+
from flax import linen
|
|
138
|
+
import jax
|
|
139
|
+
import optax
|
|
140
|
+
|
|
141
|
+
import reax
|
|
142
142
|
from reax.demos import mnist
|
|
143
143
|
|
|
144
144
|
|
|
145
145
|
class Autoencoder(linen.Module):
|
|
146
146
|
def setup(self):
|
|
147
|
-
super().__init__()
|
|
148
147
|
self.encoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(3)])
|
|
149
148
|
self.decoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(28 * 28)])
|
|
150
149
|
|
|
151
150
|
def __call__(self, x):
|
|
152
|
-
|
|
153
|
-
return self.decoder(z)
|
|
151
|
+
return self.decoder(self.encoder(x))
|
|
154
152
|
|
|
155
153
|
|
|
156
154
|
# --------------------------------
|
|
157
155
|
# Step 1: Define a REAX Module
|
|
158
156
|
# --------------------------------
|
|
159
|
-
# A
|
|
160
|
-
#
|
|
157
|
+
# A reax.Module defines a full *system* (e.g. an LLM, diffusion model, autoencoder or
|
|
158
|
+
# image classifier): the model, how to train it and how to optimise it.
|
|
161
159
|
class ReaxAutoEncoder(reax.Module):
|
|
162
160
|
def __init__(self):
|
|
163
161
|
super().__init__()
|
|
@@ -165,64 +163,65 @@ Define the training workflow. Here's a toy example:
|
|
|
165
163
|
|
|
166
164
|
def configure_model(self, stage: reax.Stage, batch, /):
|
|
167
165
|
if self.parameters() is None:
|
|
168
|
-
#
|
|
166
|
+
# Initialise the parameters using an example batch
|
|
169
167
|
x, _ = batch
|
|
170
|
-
|
|
171
|
-
# Initialize Flax Linen model with RNGs and example input
|
|
172
|
-
params = self.ae.init(self.rngs(), x)
|
|
168
|
+
params = self.ae.init(self.rngs(), x.reshape(x.shape[0], -1))
|
|
173
169
|
self.set_parameters(params)
|
|
174
170
|
|
|
175
171
|
def __call__(self, x):
|
|
176
|
-
# Use the full autoencoder model for forward pass
|
|
177
172
|
return self.ae.apply(self.parameters(), x)
|
|
178
173
|
|
|
179
174
|
def training_step(self, batch, batch_idx):
|
|
180
175
|
x, _ = batch
|
|
181
176
|
x = x.reshape(x.shape[0], -1)
|
|
182
|
-
|
|
183
|
-
loss, grads = jax.value_and_grad(self.loss_fn, argnums=0)(
|
|
184
|
-
self.parameters(), x, self.ae.apply
|
|
185
|
-
)
|
|
177
|
+
loss, grads = jax.value_and_grad(self.loss_fn)(self.parameters(), x, self.ae.apply)
|
|
186
178
|
self.log("train_loss", loss, on_step=True, prog_bar=True)
|
|
179
|
+
# Return the gradients and REAX takes care of the optimiser update
|
|
187
180
|
return loss, grads
|
|
188
181
|
|
|
182
|
+
def validation_step(self, batch, batch_idx):
|
|
183
|
+
x, _ = batch
|
|
184
|
+
x = x.reshape(x.shape[0], -1)
|
|
185
|
+
self.log("val_loss", self.loss_fn(self.parameters(), x, self.ae.apply), prog_bar=True)
|
|
186
|
+
|
|
189
187
|
@staticmethod
|
|
190
188
|
@partial(jax.jit, static_argnums=2)
|
|
191
189
|
def loss_fn(params, x, apply_fn):
|
|
192
|
-
"""Compute reconstruction loss.
|
|
193
|
-
|
|
194
|
-
Static method for JIT compilation. Receives parameters and apply function.
|
|
195
|
-
"""
|
|
196
190
|
predictions = apply_fn(params, x)
|
|
197
191
|
return optax.losses.squared_error(predictions, x).mean()
|
|
198
192
|
|
|
199
193
|
def configure_optimizers(self):
|
|
200
194
|
opt = optax.adam(learning_rate=1e-3)
|
|
201
|
-
|
|
202
|
-
return opt, state
|
|
195
|
+
return opt, opt.init(self.parameters())
|
|
203
196
|
|
|
204
197
|
|
|
205
198
|
# -------------------
|
|
206
199
|
# Step 2: Define data
|
|
207
200
|
# -------------------
|
|
208
|
-
dataset = mnist.MnistDataset(download=True)
|
|
209
201
|
trainer = reax.Trainer()
|
|
210
|
-
|
|
202
|
+
dataset = mnist.MnistDataset(download=True)
|
|
203
|
+
train, val = reax.data.random_split(trainer.rngs, dataset, [55_000, 5_000])
|
|
211
204
|
|
|
212
205
|
# -------------------
|
|
213
206
|
# Step 3: Train
|
|
214
207
|
# -------------------
|
|
215
208
|
autoencoder = ReaxAutoEncoder()
|
|
216
|
-
trainer.fit(
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
209
|
+
trainer.fit(
|
|
210
|
+
autoencoder,
|
|
211
|
+
reax.ReaxDataLoader(train, batch_size=64, shuffle=True),
|
|
212
|
+
reax.ReaxDataLoader(val, batch_size=64),
|
|
213
|
+
max_epochs=5,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
More examples, including Flax NNX and Equinox models, are in the
|
|
217
|
+
`documentation <https://reax.readthedocs.io/>`_.
|
|
218
|
+
|
|
219
|
+
Coming from PyTorch Lightning?
|
|
220
|
+
------------------------------
|
|
221
|
+
|
|
222
|
+
REAX ports the abstractions and best practices of PyTorch Lightning to JAX, and parts of its
|
|
223
|
+
code are adapted directly from Lightning. If you know Lightning, the ``Trainer``, ``Module``,
|
|
224
|
+
hooks, listeners (Lightning's callbacks) and loggers will feel familiar. Where it matters, REAX
|
|
225
|
+
departs from Lightning to fit JAX: parameters and optimiser state are explicit, and training steps
|
|
226
|
+
return gradients rather than relying on ``loss.backward()``.
|
|
228
227
|
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
REAX
|
|
2
|
+
====
|
|
3
|
+
|
|
4
|
+
.. image:: https://codecov.io/gh/muhrin/reax/branch/develop/graph/badge.svg
|
|
5
|
+
:target: https://codecov.io/gh/muhrin/reax
|
|
6
|
+
:alt: Coverage
|
|
7
|
+
|
|
8
|
+
.. image:: https://github.com/camml-lab/reax/actions/workflows/ci.yml/badge.svg
|
|
9
|
+
:target: https://github.com/camml-lab/reax/actions/workflows/ci.yml
|
|
10
|
+
:alt: Tests
|
|
11
|
+
|
|
12
|
+
.. image:: https://readthedocs.org/projects/reax/badge/
|
|
13
|
+
:target: https://reax.readthedocs.io/
|
|
14
|
+
:alt: Documentation
|
|
15
|
+
|
|
16
|
+
.. image:: https://img.shields.io/pypi/v/reax.svg
|
|
17
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
18
|
+
:alt: Latest Version
|
|
19
|
+
|
|
20
|
+
.. image:: https://img.shields.io/pypi/pyversions/reax.svg
|
|
21
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
22
|
+
|
|
23
|
+
.. image:: https://img.shields.io/pypi/l/reax.svg
|
|
24
|
+
:target: https://pypi.python.org/pypi/reax/
|
|
25
|
+
|
|
26
|
+
.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
|
|
27
|
+
:target: https://github.com/astral-sh/ruff
|
|
28
|
+
:alt: Ruff
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
**Scalable, flexible training for JAX, inspired by the simplicity of PyTorch Lightning.**
|
|
32
|
+
|
|
33
|
+
REAX is a lightweight framework for training JAX models. It takes care of the training loop,
|
|
34
|
+
logging, checkpointing and scaling across devices, so that your code can focus on the model and
|
|
35
|
+
the science. 🚀
|
|
36
|
+
|
|
37
|
+
Why REAX? 🌟
|
|
38
|
+
------------
|
|
39
|
+
|
|
40
|
+
- **Bring your own model** 🔧: REAX works with any JAX library, e.g. Flax Linen, Flax NNX or
|
|
41
|
+
Equinox. Your model stays plain JAX code.
|
|
42
|
+
- **Minimal boilerplate** 🧩: write a ``reax.Module`` with your training step and optimiser, and
|
|
43
|
+
let the ``reax.Trainer`` run fitting, validation, testing and prediction.
|
|
44
|
+
- **Batteries included** 💾: checkpointing, early stopping, progress bars, metrics, and loggers
|
|
45
|
+
for CSV, TensorBoard and MLflow.
|
|
46
|
+
- **Scales with JAX** ⚡: run on multiple devices and processes with minimal code changes.
|
|
47
|
+
- **Stay in control** 🎯: hook into every stage of training with module hooks and listeners, or
|
|
48
|
+
drop down to the ``reax.Engine`` to write your own training loop.
|
|
49
|
+
|
|
50
|
+
Installation 🛠️
|
|
51
|
+
---------------
|
|
52
|
+
|
|
53
|
+
With pip:
|
|
54
|
+
|
|
55
|
+
.. code-block:: shell
|
|
56
|
+
|
|
57
|
+
pip install reax
|
|
58
|
+
|
|
59
|
+
or with `uv <https://docs.astral.sh/uv/>`_, to add REAX to your project:
|
|
60
|
+
|
|
61
|
+
.. code-block:: shell
|
|
62
|
+
|
|
63
|
+
uv add reax
|
|
64
|
+
|
|
65
|
+
Example
|
|
66
|
+
-------
|
|
67
|
+
|
|
68
|
+
Train an autoencoder on MNIST, using a Flax Linen model:
|
|
69
|
+
|
|
70
|
+
.. code-block:: python
|
|
71
|
+
|
|
72
|
+
from functools import partial
|
|
73
|
+
|
|
74
|
+
from flax import linen
|
|
75
|
+
import jax
|
|
76
|
+
import optax
|
|
77
|
+
|
|
78
|
+
import reax
|
|
79
|
+
from reax.demos import mnist
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class Autoencoder(linen.Module):
|
|
83
|
+
def setup(self):
|
|
84
|
+
self.encoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(3)])
|
|
85
|
+
self.decoder = linen.Sequential([linen.Dense(128), linen.relu, linen.Dense(28 * 28)])
|
|
86
|
+
|
|
87
|
+
def __call__(self, x):
|
|
88
|
+
return self.decoder(self.encoder(x))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# --------------------------------
|
|
92
|
+
# Step 1: Define a REAX Module
|
|
93
|
+
# --------------------------------
|
|
94
|
+
# A reax.Module defines a full *system* (e.g. an LLM, diffusion model, autoencoder or
|
|
95
|
+
# image classifier): the model, how to train it and how to optimise it.
|
|
96
|
+
class ReaxAutoEncoder(reax.Module):
|
|
97
|
+
def __init__(self):
|
|
98
|
+
super().__init__()
|
|
99
|
+
self.ae = Autoencoder()
|
|
100
|
+
|
|
101
|
+
def configure_model(self, stage: reax.Stage, batch, /):
|
|
102
|
+
if self.parameters() is None:
|
|
103
|
+
# Initialise the parameters using an example batch
|
|
104
|
+
x, _ = batch
|
|
105
|
+
params = self.ae.init(self.rngs(), x.reshape(x.shape[0], -1))
|
|
106
|
+
self.set_parameters(params)
|
|
107
|
+
|
|
108
|
+
def __call__(self, x):
|
|
109
|
+
return self.ae.apply(self.parameters(), x)
|
|
110
|
+
|
|
111
|
+
def training_step(self, batch, batch_idx):
|
|
112
|
+
x, _ = batch
|
|
113
|
+
x = x.reshape(x.shape[0], -1)
|
|
114
|
+
loss, grads = jax.value_and_grad(self.loss_fn)(self.parameters(), x, self.ae.apply)
|
|
115
|
+
self.log("train_loss", loss, on_step=True, prog_bar=True)
|
|
116
|
+
# Return the gradients and REAX takes care of the optimiser update
|
|
117
|
+
return loss, grads
|
|
118
|
+
|
|
119
|
+
def validation_step(self, batch, batch_idx):
|
|
120
|
+
x, _ = batch
|
|
121
|
+
x = x.reshape(x.shape[0], -1)
|
|
122
|
+
self.log("val_loss", self.loss_fn(self.parameters(), x, self.ae.apply), prog_bar=True)
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
@partial(jax.jit, static_argnums=2)
|
|
126
|
+
def loss_fn(params, x, apply_fn):
|
|
127
|
+
predictions = apply_fn(params, x)
|
|
128
|
+
return optax.losses.squared_error(predictions, x).mean()
|
|
129
|
+
|
|
130
|
+
def configure_optimizers(self):
|
|
131
|
+
opt = optax.adam(learning_rate=1e-3)
|
|
132
|
+
return opt, opt.init(self.parameters())
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# -------------------
|
|
136
|
+
# Step 2: Define data
|
|
137
|
+
# -------------------
|
|
138
|
+
trainer = reax.Trainer()
|
|
139
|
+
dataset = mnist.MnistDataset(download=True)
|
|
140
|
+
train, val = reax.data.random_split(trainer.rngs, dataset, [55_000, 5_000])
|
|
141
|
+
|
|
142
|
+
# -------------------
|
|
143
|
+
# Step 3: Train
|
|
144
|
+
# -------------------
|
|
145
|
+
autoencoder = ReaxAutoEncoder()
|
|
146
|
+
trainer.fit(
|
|
147
|
+
autoencoder,
|
|
148
|
+
reax.ReaxDataLoader(train, batch_size=64, shuffle=True),
|
|
149
|
+
reax.ReaxDataLoader(val, batch_size=64),
|
|
150
|
+
max_epochs=5,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
More examples, including Flax NNX and Equinox models, are in the
|
|
154
|
+
`documentation <https://reax.readthedocs.io/>`_.
|
|
155
|
+
|
|
156
|
+
Coming from PyTorch Lightning?
|
|
157
|
+
------------------------------
|
|
158
|
+
|
|
159
|
+
REAX ports the abstractions and best practices of PyTorch Lightning to JAX, and parts of its
|
|
160
|
+
code are adapted directly from Lightning. If you know Lightning, the ``Trainer``, ``Module``,
|
|
161
|
+
hooks, listeners (Lightning's callbacks) and loggers will feel familiar. Where it matters, REAX
|
|
162
|
+
departs from Lightning to fit JAX: parameters and optimiser state are explicit, and training steps
|
|
163
|
+
return gradients rather than relying on ``loss.backward()``.
|
|
@@ -60,6 +60,7 @@ dev = [
|
|
|
60
60
|
'flit',
|
|
61
61
|
'ipython',
|
|
62
62
|
"mlflow<3.7",
|
|
63
|
+
"omegaconf",
|
|
63
64
|
'pandas',
|
|
64
65
|
'pytest',
|
|
65
66
|
'pytest-cov',
|
|
@@ -68,6 +69,7 @@ dev = [
|
|
|
68
69
|
'pre-commit',
|
|
69
70
|
'pylint',
|
|
70
71
|
'ruff',
|
|
72
|
+
'ty==0.0.84',
|
|
71
73
|
]
|
|
72
74
|
examples = [
|
|
73
75
|
"lightning",
|
|
@@ -119,8 +121,6 @@ select = [
|
|
|
119
121
|
ignore = [
|
|
120
122
|
'E203',
|
|
121
123
|
'E266',
|
|
122
|
-
'E501',
|
|
123
|
-
'F401',
|
|
124
124
|
'F403',
|
|
125
125
|
'F722',
|
|
126
126
|
'F821',
|
|
@@ -140,11 +140,84 @@ max-complexity = 18
|
|
|
140
140
|
'C90',
|
|
141
141
|
]
|
|
142
142
|
|
|
143
|
+
[tool.ty.src]
|
|
144
|
+
# TODO: These files have outstanding ty errors, remove them from this list as they are fixed
|
|
145
|
+
exclude = [
|
|
146
|
+
'docs/source/get_started/introduction.ipynb',
|
|
147
|
+
'docs/source/get_started/quick_start.ipynb',
|
|
148
|
+
'examples/autoencoder.py',
|
|
149
|
+
'src/reax/data/_datasource_manager.py',
|
|
150
|
+
'src/reax/data/_types.py',
|
|
151
|
+
'src/reax/data/collate.py',
|
|
152
|
+
'src/reax/data/datamodules.py',
|
|
153
|
+
'src/reax/data/datasets.py',
|
|
154
|
+
'src/reax/data/fetchers.py',
|
|
155
|
+
'src/reax/data/utils.py',
|
|
156
|
+
'src/reax/demos/boring_classes.py',
|
|
157
|
+
'src/reax/demos/mnist.py',
|
|
158
|
+
'src/reax/listeners/early_stopping.py',
|
|
159
|
+
'src/reax/listeners/model_checkpoint.py',
|
|
160
|
+
'src/reax/listeners/progress/tqdm_progress.py',
|
|
161
|
+
'src/reax/loggers/_utils.py',
|
|
162
|
+
'src/reax/loggers/csv_logs.py',
|
|
163
|
+
'src/reax/loggers/logger.py',
|
|
164
|
+
'src/reax/loggers/mlflow.py',
|
|
165
|
+
'src/reax/loggers/pandas.py',
|
|
166
|
+
'src/reax/loggers/tensorboard.py',
|
|
167
|
+
'src/reax/modules.py',
|
|
168
|
+
'src/reax/optimizers.py',
|
|
169
|
+
'src/reax/random.py',
|
|
170
|
+
'src/reax/results.py',
|
|
171
|
+
'src/reax/strategies/_factory.py',
|
|
172
|
+
'src/reax/strategies/_jax.py',
|
|
173
|
+
'src/reax/strategies/_parallel.py',
|
|
174
|
+
'src/reax/training/trainer.py',
|
|
175
|
+
'src/reax/utils/arrays.py',
|
|
176
|
+
'src/reax/utils/containers.py',
|
|
177
|
+
'src/reax/utils/events.py',
|
|
178
|
+
'tests/checkpointing/test_checkpoint_callback_frequency.py',
|
|
179
|
+
'tests/core/test_module.py',
|
|
180
|
+
'tests/core/todo_test_module.py',
|
|
181
|
+
'tests/data/test_collate.py',
|
|
182
|
+
'tests/data/test_datamodules_and_manager.py',
|
|
183
|
+
'tests/data/test_datasets.py',
|
|
184
|
+
'tests/data/test_fetchers.py',
|
|
185
|
+
'tests/data/test_kfold.py',
|
|
186
|
+
'tests/data/test_kfold_distributed.py',
|
|
187
|
+
'tests/data/test_loaders.py',
|
|
188
|
+
'tests/data/test_samplers.py',
|
|
189
|
+
'tests/data/test_utils.py',
|
|
190
|
+
'tests/engine/test_engine_ddp.py',
|
|
191
|
+
'tests/helpers/simple_models.py',
|
|
192
|
+
'tests/listeners/progress/test_rich_progress_bar.py',
|
|
193
|
+
'tests/listeners/test_early_stopping.py',
|
|
194
|
+
'tests/listeners/test_listeners.py',
|
|
195
|
+
'tests/listeners/test_model_checkpoint.py',
|
|
196
|
+
'tests/loggers/conftest.py',
|
|
197
|
+
'tests/loggers/test_mlflow.py',
|
|
198
|
+
'tests/loggers/test_pandas.py',
|
|
199
|
+
'tests/loggers/test_tensorboard.py',
|
|
200
|
+
'tests/models/test_hooks.py',
|
|
201
|
+
'tests/profilers/test_jax_profiler.py',
|
|
202
|
+
'tests/stages/test_timer.py',
|
|
203
|
+
'tests/strategies/test_parallel_strategy.py',
|
|
204
|
+
'tests/test_metrics.py',
|
|
205
|
+
'tests/training/test_dataloaders.py',
|
|
206
|
+
'tests/training/test_trainer.py',
|
|
207
|
+
'tests/utils/test_arrays.py',
|
|
208
|
+
'tests/utils/test_containers.py',
|
|
209
|
+
]
|
|
210
|
+
|
|
211
|
+
[tool.ty.analysis]
|
|
212
|
+
allowed-unresolved-imports = ["torch.**"]
|
|
213
|
+
|
|
214
|
+
|
|
143
215
|
[tool.pylint.MASTER]
|
|
144
216
|
ignore-paths = '^src/reax/lightning/.*$' # These are copy-paste from lightning, so don't lint
|
|
145
217
|
|
|
146
218
|
[tool.pylint.format]
|
|
147
219
|
max-line-length = 100
|
|
220
|
+
max-module-lines = 1200
|
|
148
221
|
|
|
149
222
|
[tool.pylint.messages_control]
|
|
150
223
|
disable = [
|
|
@@ -16,6 +16,7 @@ from . import hooks, modules, optimizers, profilers, strategies
|
|
|
16
16
|
from . import loggers as loggers_
|
|
17
17
|
from . import metrics as metrics_
|
|
18
18
|
from .data import DeviceDataLoader
|
|
19
|
+
from .lightning import rank_zero
|
|
19
20
|
from .training import _logger_connector
|
|
20
21
|
from .utils import events
|
|
21
22
|
|
|
@@ -63,7 +64,7 @@ class Engine:
|
|
|
63
64
|
rngs: nnx.Rngs | None = None,
|
|
64
65
|
default_root_dir: "reax.types.Path | None" = None,
|
|
65
66
|
profiler: "reax.Profiler | str | None" = None,
|
|
66
|
-
metric_evaluator: "reax.MetricEvaluator | None" = None,
|
|
67
|
+
metric_evaluator: "reax.metrics.MetricEvaluator | None" = None,
|
|
67
68
|
):
|
|
68
69
|
"""
|
|
69
70
|
Initializes the Engine with execution parameters and components.
|
|
@@ -105,6 +106,10 @@ class Engine:
|
|
|
105
106
|
self._strategy = strategy
|
|
106
107
|
else:
|
|
107
108
|
self._strategy = strategies.create(strategy, accelerator, devices=devices)
|
|
109
|
+
# The strategy knows this process's rank whatever launched it (e.g. REAX itself, MPI or
|
|
110
|
+
# SLURM), so make the rank-zero utilities agree with it. This has to happen before the
|
|
111
|
+
# loggers are created, as they only write from rank zero.
|
|
112
|
+
rank_zero.set_rank(self.process_index)
|
|
108
113
|
self._default_root_dir = (
|
|
109
114
|
os.fspath(default_root_dir) if default_root_dir is not None else os.getcwd()
|
|
110
115
|
)
|
|
@@ -132,9 +137,10 @@ class Engine:
|
|
|
132
137
|
After this is called, this object should no longer be interacted with,
|
|
133
138
|
as internal components (like event listeners and loggers) are cleared.
|
|
134
139
|
"""
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
self.
|
|
140
|
+
# Note: avoid referencing module globals here, as this can be called from `Trainer.__del__`
|
|
141
|
+
# during interpreter shutdown, when they may already have been cleared
|
|
142
|
+
self._events.reset()
|
|
143
|
+
self._loggers = []
|
|
138
144
|
|
|
139
145
|
@property
|
|
140
146
|
def default_root_dir(self) -> str:
|
|
@@ -466,7 +472,7 @@ def _(arg: data_.DataLoader, engine: Engine) -> data_.DataLoader:
|
|
|
466
472
|
def _init_loggers(
|
|
467
473
|
logger: "reax.Logger | Iterable[reax.Logger] | bool | None",
|
|
468
474
|
default_root_dir: str,
|
|
469
|
-
) -> list[
|
|
475
|
+
) -> "list[reax.Logger]":
|
|
470
476
|
"""Init loggers."""
|
|
471
477
|
if isinstance(logger, loggers_.Logger):
|
|
472
478
|
return [logger]
|
|
@@ -481,7 +487,7 @@ def _init_loggers(
|
|
|
481
487
|
|
|
482
488
|
|
|
483
489
|
def _init_profiler(profiler: "reax.Profiler | str | None" = None) -> "reax.Profiler":
|
|
484
|
-
profs: Final[dict[str, type
|
|
490
|
+
profs: Final[dict[str, type[profilers.Profiler]]] = {
|
|
485
491
|
"dummy": profilers.DummyProfiler,
|
|
486
492
|
"jax": profilers.JaxProfiler,
|
|
487
493
|
}
|