reax 0.7.0.dev5__tar.gz → 0.7.0.dev6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/PKG-INFO +5 -4
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/pyproject.toml +6 -5
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/__init__.py +1 -2
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_engine.py +62 -11
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_module_hooks.py +4 -4
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_datasources.py +2 -17
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/datasets.py +15 -11
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/fetchers.py +4 -2
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/kfold.py +5 -20
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/samplers.py +53 -10
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/boring_classes.py +1 -1
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/early_stopping.py +6 -4
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/model_checkpoint.py +22 -22
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/progress_bar.py +2 -4
- reax-0.7.0.dev6/src/reax/loggers/__init__.py +14 -0
- reax-0.7.0.dev6/src/reax/loggers/_figures.py +219 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/logger.py +131 -2
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/mlflow.py +153 -16
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/tensorboard.py +17 -1
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_classification.py +19 -23
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_metric.py +2 -2
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/aggregation.py +2 -2
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/modules.py +47 -5
- reax-0.7.0.dev6/src/reax/random.py +156 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_checkpointing.py +5 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/trainer.py +22 -4
- reax-0.7.0.dev5/src/reax/loggers/__init__.py +0 -7
- reax-0.7.0.dev5/src/reax/random.py +0 -22
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/LICENSE.txt +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/README.rst +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_vendor/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_vendor/simple_pytree.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_datasource_manager.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_loaders.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_types.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/collate.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/dataloaders.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/datamodules.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/mnist.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/exceptions.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/hooks.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/keys.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/LICENSE +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/rank_zero.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/checkpointer.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/rich_progress.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/tqdm_progress.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/_utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/csv_logs.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/pandas.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_evaluators.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_globals.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_registry.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/collections.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/types.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/regression.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/optimizers.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/plugins.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/jax_profiler.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/profiler.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/pytree.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/results.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/saving.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/_loops.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/_timer.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/common.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/fit.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/predict.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/stages.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/stats.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/test.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/train.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/validation.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_factory.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_jax.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_parallel.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_single_device.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_strategies.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/testing.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_checkpoint_connector.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_hooks_map.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_logger_connector.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_utils.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/types.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/__init__.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/arrays.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/containers.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/events.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/grads.py +0 -0
- {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/masking.py +0 -0
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: reax
|
|
3
|
-
Version: 0.7.0.
|
|
3
|
+
Version: 0.7.0.dev6
|
|
4
4
|
Summary: REAX: A simple training framework for JAX-based projects
|
|
5
5
|
Keywords: machine learning,jax,research
|
|
6
6
|
Author-email: Martin Uhrin <martin.uhrin.10@ucl.ac.uk>
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
8
|
Description-Content-Type: text/x-rst
|
|
9
9
|
Classifier: Development Status :: 4 - Beta
|
|
10
10
|
Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
@@ -32,9 +31,11 @@ Requires-Dist: typing-extensions
|
|
|
32
31
|
Requires-Dist: cloudpickle ; extra == "dev"
|
|
33
32
|
Requires-Dist: flit ; extra == "dev"
|
|
34
33
|
Requires-Dist: ipython ; extra == "dev"
|
|
35
|
-
Requires-Dist:
|
|
34
|
+
Requires-Dist: matplotlib ; extra == "dev"
|
|
35
|
+
Requires-Dist: mlflow>=3.15 ; extra == "dev"
|
|
36
36
|
Requires-Dist: omegaconf ; extra == "dev"
|
|
37
37
|
Requires-Dist: pandas ; extra == "dev"
|
|
38
|
+
Requires-Dist: pillow ; extra == "dev"
|
|
38
39
|
Requires-Dist: pytest ; extra == "dev"
|
|
39
40
|
Requires-Dist: pytest-cov ; extra == "dev"
|
|
40
41
|
Requires-Dist: pytest-env ; extra == "dev"
|
|
@@ -14,14 +14,13 @@ classifiers = [
|
|
|
14
14
|
'Development Status :: 4 - Beta',
|
|
15
15
|
'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
|
|
16
16
|
'Programming Language :: Python',
|
|
17
|
-
'Programming Language :: Python :: 3.10',
|
|
18
17
|
'Programming Language :: Python :: 3.11',
|
|
19
18
|
'Programming Language :: Python :: 3.12',
|
|
20
19
|
'Programming Language :: Python :: 3.13',
|
|
21
20
|
'Programming Language :: Python :: 3.14',
|
|
22
21
|
]
|
|
23
22
|
keywords = ['machine learning', 'jax', 'research']
|
|
24
|
-
requires-python = '>=3.
|
|
23
|
+
requires-python = '>=3.11'
|
|
25
24
|
dependencies = [
|
|
26
25
|
'beartype',
|
|
27
26
|
"einops",
|
|
@@ -59,9 +58,11 @@ dev = [
|
|
|
59
58
|
'cloudpickle',
|
|
60
59
|
'flit',
|
|
61
60
|
'ipython',
|
|
62
|
-
|
|
61
|
+
'matplotlib',
|
|
62
|
+
"mlflow>=3.15",
|
|
63
63
|
"omegaconf",
|
|
64
64
|
'pandas',
|
|
65
|
+
'pillow',
|
|
65
66
|
'pytest',
|
|
66
67
|
'pytest-cov',
|
|
67
68
|
'pytest-env',
|
|
@@ -101,7 +102,7 @@ mae = "reax.metrics:MeanAbsoluteError"
|
|
|
101
102
|
|
|
102
103
|
[tool.ruff]
|
|
103
104
|
line-length = 100
|
|
104
|
-
target-version = "
|
|
105
|
+
target-version = "py311"
|
|
105
106
|
exclude = [
|
|
106
107
|
'migrations',
|
|
107
108
|
]
|
|
@@ -280,7 +281,7 @@ envlist = py311
|
|
|
280
281
|
[testenv]
|
|
281
282
|
usedevelop = true
|
|
282
283
|
|
|
283
|
-
[testenv:py{
|
|
284
|
+
[testenv:py{311,312,313,314}]
|
|
284
285
|
description = Run the unit tests
|
|
285
286
|
extras =
|
|
286
287
|
dev
|
|
@@ -10,11 +10,13 @@ from flax import nnx
|
|
|
10
10
|
import fsspec.utils
|
|
11
11
|
import jax
|
|
12
12
|
import jaxtyping as jt
|
|
13
|
+
import numpy as np
|
|
13
14
|
|
|
14
15
|
from . import data as data_
|
|
15
16
|
from . import hooks, modules, optimizers, profilers, strategies
|
|
16
17
|
from . import loggers as loggers_
|
|
17
18
|
from . import metrics as metrics_
|
|
19
|
+
from . import random as random_
|
|
18
20
|
from .data import DeviceDataLoader
|
|
19
21
|
from .lightning import rank_zero
|
|
20
22
|
from .training import _logger_connector
|
|
@@ -43,10 +45,6 @@ class Engine:
|
|
|
43
45
|
It encapsulates the environment setup defined by the chosen strategy,
|
|
44
46
|
managing details like accelerator configuration, process launching (e.g.,
|
|
45
47
|
via MPI or Python's multiprocessing), and backend initialization.
|
|
46
|
-
|
|
47
|
-
Attributes:
|
|
48
|
-
strategy (object): The execution strategy defining the environment and distributed
|
|
49
|
-
communication.
|
|
50
48
|
"""
|
|
51
49
|
|
|
52
50
|
# pylint: disable=too-many-public-methods
|
|
@@ -65,6 +63,7 @@ class Engine:
|
|
|
65
63
|
default_root_dir: "reax.types.Path | None" = None,
|
|
66
64
|
profiler: "reax.Profiler | str | None" = None,
|
|
67
65
|
metric_evaluator: "reax.metrics.MetricEvaluator | None" = None,
|
|
66
|
+
seed: int | None = None,
|
|
68
67
|
):
|
|
69
68
|
"""
|
|
70
69
|
Initializes the Engine with execution parameters and components.
|
|
@@ -86,11 +85,18 @@ class Engine:
|
|
|
86
85
|
listeners (list[reax.TrainerListener] | reax.TrainerListener | None): A single or list
|
|
87
86
|
of :class:`reax.TrainerListener` objects for hooking into events.
|
|
88
87
|
deterministic (bool): Currently not supported, a warning is issued if set.
|
|
89
|
-
rngs (nnx.Rngs):
|
|
90
|
-
|
|
88
|
+
rngs (nnx.Rngs): Advanced: an explicit random number generator state to use instead of
|
|
89
|
+
the one REAX builds from ``seed``. Cannot be combined with ``seed``.
|
|
91
90
|
default_root_dir (typing.Path | None): The default directory path for saving logs and
|
|
92
91
|
checkpoints if not specified elsewhere. Defaults to the current working directory.
|
|
93
|
-
|
|
92
|
+
seed (int | None): The single seed from which all randomness is derived (see
|
|
93
|
+
:mod:`reax.random`). Pass an integer for a reproducible run. If ``None``, a seed
|
|
94
|
+
is drawn at random (and shared across all processes), so every run is different.
|
|
95
|
+
Either way, the seed is logged, stored in checkpoints and available as
|
|
96
|
+
:attr:`seed`, so any run can be reproduced.
|
|
97
|
+
"""
|
|
98
|
+
if rngs is not None and seed is not None:
|
|
99
|
+
raise ValueError("Pass either `seed` or `rngs`, not both.")
|
|
94
100
|
if deterministic:
|
|
95
101
|
_LOGGER.warning("`deterministic=True` is not supported yet, ignoring.")
|
|
96
102
|
if precision is not None:
|
|
@@ -116,8 +122,17 @@ class Engine:
|
|
|
116
122
|
self._events = events.EventGenerator[hooks.TrainerListener](
|
|
117
123
|
default_args=(weakref.proxy(self),)
|
|
118
124
|
)
|
|
119
|
-
self._rngs = rngs if rngs is not None else nnx.Rngs(0)
|
|
120
125
|
self._loggers: list[loggers_.Logger] = _init_loggers(logger, self.default_root_dir)
|
|
126
|
+
self._seed: int | None
|
|
127
|
+
if rngs is not None:
|
|
128
|
+
self._seed = None
|
|
129
|
+
self._rngs = rngs
|
|
130
|
+
else:
|
|
131
|
+
self._seed = self._resolve_seed(seed)
|
|
132
|
+
self._rngs = random_.create_rngs(self._seed, process_index=self.process_index)
|
|
133
|
+
random_.seed_globals(self._seed)
|
|
134
|
+
if self.is_global_zero:
|
|
135
|
+
_LOGGER.info("Using seed: %i", self._seed)
|
|
121
136
|
self._logging = _logger_connector.TrainerLogging()
|
|
122
137
|
self._profiler: profilers.Profiler = _init_profiler(profiler)
|
|
123
138
|
|
|
@@ -192,13 +207,35 @@ class Engine:
|
|
|
192
207
|
@property
|
|
193
208
|
def rngs(self) -> nnx.Rngs:
|
|
194
209
|
"""
|
|
195
|
-
The
|
|
210
|
+
The random number generators from which all randomness should be drawn.
|
|
211
|
+
|
|
212
|
+
These have independent named streams (see :mod:`reax.random`): ``params``, ``data`` and
|
|
213
|
+
``default`` are identical on all processes, while ``dropout`` differs between processes.
|
|
196
214
|
|
|
197
215
|
Returns:
|
|
198
216
|
nnx.Rngs: The RNG state object.
|
|
199
217
|
"""
|
|
200
218
|
return self._rngs
|
|
201
219
|
|
|
220
|
+
@property
|
|
221
|
+
def seed(self) -> int | None:
|
|
222
|
+
"""
|
|
223
|
+
The seed from which :attr:`rngs` were created, or ``None`` if explicit ``rngs`` were
|
|
224
|
+
passed in.
|
|
225
|
+
"""
|
|
226
|
+
return self._seed
|
|
227
|
+
|
|
228
|
+
def _resolve_seed(self, seed: int | None) -> int:
|
|
229
|
+
"""Get the seed to use, drawing a random one (shared by all processes) if needed."""
|
|
230
|
+
if seed is not None:
|
|
231
|
+
return int(seed)
|
|
232
|
+
|
|
233
|
+
seed = random_.draw_seed()
|
|
234
|
+
if self.process_count > 1:
|
|
235
|
+
seed = int(self.broadcast(np.asarray(seed, dtype=np.int32), src=0))
|
|
236
|
+
|
|
237
|
+
return seed
|
|
238
|
+
|
|
202
239
|
@property
|
|
203
240
|
def strategy(self) -> strategies.Strategy:
|
|
204
241
|
"""
|
|
@@ -339,7 +376,8 @@ class Engine:
|
|
|
339
376
|
reax.data.DeviceDataLoader | list[reax.data.DeviceDataLoader]: The prepared
|
|
340
377
|
dataloader(s), wrapped as a :class:`~reax.data.DeviceDataLoader`.
|
|
341
378
|
"""
|
|
342
|
-
loaders = list(map(self.
|
|
379
|
+
loaders = list(map(self._seed_loader, args))
|
|
380
|
+
loaders = list(map(self.strategy.setup_dataloader, loaders))
|
|
343
381
|
# Wrap in DeviceLoader so all data is already on the selected device
|
|
344
382
|
loaders = list(map(self._wrap_loader, loaders))
|
|
345
383
|
if len(args) == 1:
|
|
@@ -436,6 +474,19 @@ class Engine:
|
|
|
436
474
|
"""
|
|
437
475
|
return self._strategy.compute(metric)
|
|
438
476
|
|
|
477
|
+
def _seed_loader(self, loader: "reax.DataLoader") -> "reax.DataLoader":
|
|
478
|
+
"""Give any unseeded :class:`~reax.data.samplers.RandomSampler` used by the loader a seed
|
|
479
|
+
drawn from the ``data`` stream, so shuffling follows the engine seed and is the same on
|
|
480
|
+
all processes."""
|
|
481
|
+
inner = loader
|
|
482
|
+
while isinstance(inner, DeviceDataLoader):
|
|
483
|
+
inner = inner.parent
|
|
484
|
+
for sub in data_.samplers.iter_samplers(getattr(inner, "sampler", None)):
|
|
485
|
+
if isinstance(sub, data_.samplers.RandomSampler) and sub.seed is None:
|
|
486
|
+
sub.seed = random_.key_to_seed(self._rngs.data())
|
|
487
|
+
|
|
488
|
+
return loader
|
|
489
|
+
|
|
439
490
|
def _wrap_loader(self, loader: "reax.DataLoader") -> "reax.data.DeviceDataLoader":
|
|
440
491
|
"""
|
|
441
492
|
Internal utility to wrap a standard DataLoader into a DeviceDataLoader.
|
|
@@ -465,7 +516,7 @@ def _(arg: modules.Module, engine: Engine) -> modules.Module:
|
|
|
465
516
|
|
|
466
517
|
@_setup.register
|
|
467
518
|
def _(arg: data_.DataLoader, engine: Engine) -> data_.DataLoader:
|
|
468
|
-
return engine.strategy.setup_dataloader(arg)
|
|
519
|
+
return engine.strategy.setup_dataloader(engine._seed_loader(arg)) # pylint: disable=protected-access
|
|
469
520
|
|
|
470
521
|
|
|
471
522
|
@jt.jaxtyped(typechecker=beartype.beartype)
|
|
@@ -300,10 +300,10 @@ class ModuleHooks:
|
|
|
300
300
|
|
|
301
301
|
Only called when using automatic optimization (see
|
|
302
302
|
:attr:`~reax.Module.automatic_optimization`). This is called for every training batch,
|
|
303
|
-
including when gradients are being accumulated (see
|
|
304
|
-
:
|
|
305
|
-
the optimizer. Any transformations that are part of the optimizer (e.g.
|
|
306
|
-
clipping) will not have been applied yet.
|
|
303
|
+
including when gradients are being accumulated (see the
|
|
304
|
+
``accumulate_grad_batches`` argument of :meth:`~reax.Trainer.fit`), as accumulation is
|
|
305
|
+
performed by the optimizer. Any transformations that are part of the optimizer (e.g.
|
|
306
|
+
gradient clipping) will not have been applied yet.
|
|
307
307
|
|
|
308
308
|
Args:
|
|
309
309
|
optimizer: The optimizer that is about to be updated.
|
|
@@ -52,7 +52,7 @@ class DataSource(Generic[_T_co, U]):
|
|
|
52
52
|
self.some_state = some_other_state()
|
|
53
53
|
|
|
54
54
|
In a distributed environment, ``prepare_data`` can be called in two ways
|
|
55
|
-
(using :
|
|
55
|
+
(using :attr:`prepare_data_per_node`)
|
|
56
56
|
|
|
57
57
|
1. Once per node. This is the default and is only called on LOCAL_RANK=0.
|
|
58
58
|
2. Once in total. Only called on GLOBAL_RANK=0.
|
|
@@ -135,11 +135,6 @@ class DataSource(Generic[_T_co, U]):
|
|
|
135
135
|
def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
136
136
|
"""An iterable or collection of iterables specifying training samples.
|
|
137
137
|
|
|
138
|
-
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
139
|
-
|
|
140
|
-
The dataloader you return will not be reloaded unless you set
|
|
141
|
-
:paramref:`~reax.Trainer.fit(reload_dataloaders_every_n_epochs=k)` to a positive integer.
|
|
142
|
-
|
|
143
138
|
For data processing use the following pattern:
|
|
144
139
|
|
|
145
140
|
- download in :meth:`prepare_data`
|
|
@@ -166,12 +161,6 @@ class DataSource(Generic[_T_co, U]):
|
|
|
166
161
|
def val_dataloader(self) -> Optional["reax.DataLoader[_T_co, U]"]:
|
|
167
162
|
r"""An iterable or collection of iterables specifying validation samples.
|
|
168
163
|
|
|
169
|
-
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
170
|
-
|
|
171
|
-
The dataloader you return will not be reloaded unless you set
|
|
172
|
-
:paramref:`~lightning.pytorch.trainer.trainer.Trainer.reload_dataloaders_every_n_epochs` to
|
|
173
|
-
a positive integer.
|
|
174
|
-
|
|
175
164
|
It's recommended that all data downloads and preparation happen in :meth:`prepare_data`.
|
|
176
165
|
|
|
177
166
|
- :meth:`~reax.Trainer.fit`
|
|
@@ -192,8 +181,6 @@ class DataSource(Generic[_T_co, U]):
|
|
|
192
181
|
def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
193
182
|
r"""An iterable or collection of iterables specifying test samples.
|
|
194
183
|
|
|
195
|
-
For more information about multiple dataloaders, this :ref:`section <multiple-dataloaders>`.
|
|
196
|
-
|
|
197
184
|
For data processing use the following pattern:
|
|
198
185
|
|
|
199
186
|
- download in :meth:`prepare_data`
|
|
@@ -204,7 +191,7 @@ class DataSource(Generic[_T_co, U]):
|
|
|
204
191
|
Warning:
|
|
205
192
|
do not assign state in prepare_data
|
|
206
193
|
|
|
207
|
-
- :meth:`~
|
|
194
|
+
- :meth:`~reax.Trainer.test`
|
|
208
195
|
- :meth:`prepare_data`
|
|
209
196
|
- :meth:`setup`
|
|
210
197
|
|
|
@@ -224,8 +211,6 @@ class DataSource(Generic[_T_co, U]):
|
|
|
224
211
|
def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
225
212
|
r"""An iterable or collection of iterables specifying prediction samples.
|
|
226
213
|
|
|
227
|
-
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
228
|
-
|
|
229
214
|
It's recommended that all data downloads and preparation happen in :meth:`prepare_data`.
|
|
230
215
|
|
|
231
216
|
- :meth:`~reax.Trainer.predict`
|
|
@@ -2,17 +2,20 @@ import bisect
|
|
|
2
2
|
from collections.abc import Iterable, Sequence
|
|
3
3
|
import itertools
|
|
4
4
|
import math
|
|
5
|
-
from typing import TypeVar, cast
|
|
5
|
+
from typing import TYPE_CHECKING, TypeVar, cast
|
|
6
6
|
import warnings
|
|
7
7
|
|
|
8
8
|
import beartype
|
|
9
|
-
from flax import nnx
|
|
10
9
|
import jax.random
|
|
11
10
|
import jaxtyping as jt
|
|
12
11
|
import numpy as np
|
|
13
12
|
|
|
13
|
+
from .. import random as random_
|
|
14
14
|
from . import _types
|
|
15
15
|
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from flax import nnx
|
|
18
|
+
|
|
16
19
|
__all__ = ("ArrayDataset", "ConcatDataset", "Subset", "random_split")
|
|
17
20
|
|
|
18
21
|
_T = TypeVar("_T")
|
|
@@ -132,7 +135,7 @@ class Subset(Sequence[_T_co]):
|
|
|
132
135
|
|
|
133
136
|
|
|
134
137
|
def random_split(
|
|
135
|
-
rngs: nnx.Rngs,
|
|
138
|
+
rngs: "nnx.Rngs | nnx.RngStream | jax.Array | int",
|
|
136
139
|
dataset: Sequence[_T],
|
|
137
140
|
lengths: Sequence[int | float],
|
|
138
141
|
) -> list[Subset[_T]]:
|
|
@@ -146,19 +149,19 @@ def random_split(
|
|
|
146
149
|
distributed in round-robin fashion to the lengths
|
|
147
150
|
until there are no remainders left.
|
|
148
151
|
|
|
149
|
-
|
|
152
|
+
The split is drawn from ``rngs``; pass ``engine.rngs`` to have it follow the engine seed (and
|
|
153
|
+
be identical on all processes).
|
|
150
154
|
|
|
151
155
|
Example:
|
|
152
|
-
>>>
|
|
153
|
-
>>>
|
|
154
|
-
>>>
|
|
155
|
-
>>> random_split(key1, range(10), [3, 7])
|
|
156
|
-
>>> random_split(key2, range(30), [0.3, 0.3, 0.4])
|
|
156
|
+
>>> rngs = nnx.Rngs(data=42)
|
|
157
|
+
>>> train, val = random_split(rngs, range(10), [3, 7])
|
|
158
|
+
>>> train, val, test = random_split(jax.random.key(0), range(30), [0.3, 0.3, 0.4])
|
|
157
159
|
|
|
158
160
|
Args:
|
|
161
|
+
rngs: The source of randomness: an :class:`flax.nnx.Rngs` (a key is drawn from its
|
|
162
|
+
``data`` stream), an :class:`flax.nnx.RngStream`, a JAX key or an integer seed.
|
|
159
163
|
dataset (Dataset): Dataset to be split
|
|
160
164
|
lengths (sequence): lengths or fractions of splits to be produced
|
|
161
|
-
generator (Generator): Generator used for the random permutation.
|
|
162
165
|
"""
|
|
163
166
|
if math.isclose(sum(lengths), 1) and sum(lengths) <= 1:
|
|
164
167
|
subset_lengths: list[int] = []
|
|
@@ -184,7 +187,8 @@ def random_split(
|
|
|
184
187
|
if sum(lengths) != len(dataset):
|
|
185
188
|
raise ValueError("Sum of input lengths does not equal the length of the input dataset!")
|
|
186
189
|
|
|
187
|
-
|
|
190
|
+
key = random_.to_key(rngs, stream="data")
|
|
191
|
+
indices = jax.random.permutation(key, sum(lengths)).tolist()
|
|
188
192
|
lengths = cast(Sequence[int], lengths)
|
|
189
193
|
return [
|
|
190
194
|
Subset(dataset, indices[offset - length : offset])
|
|
@@ -5,6 +5,7 @@ import functools
|
|
|
5
5
|
from typing import Generic, TypeVar
|
|
6
6
|
|
|
7
7
|
import jax
|
|
8
|
+
import numpy as np
|
|
8
9
|
|
|
9
10
|
from . import _types
|
|
10
11
|
|
|
@@ -111,11 +112,12 @@ def create_fetcher(
|
|
|
111
112
|
CPython's singledispatch C3 linearization places the ``Iterable`` abstract base class before
|
|
112
113
|
the concrete ``jax.Array`` in the composed MRO. Registering a ``jax.Array`` handler therefore
|
|
113
114
|
would never be selected and the array would be handled by the iterable fetcher, which ignores
|
|
114
|
-
the sampler indices and yields the dataset sequentially.
|
|
115
|
+
the sampler indices and yields the dataset sequentially (``numpy.ndarray`` isn't a ``Sequence``
|
|
116
|
+
either, so has the same problem). We disambiguate with an explicit
|
|
115
117
|
``isinstance`` check (mirroring the equivalent special case in ``create_sampler``) so the array
|
|
116
118
|
is treated as a ``_MapFetcher`` and the sampler indices are actually used.
|
|
117
119
|
"""
|
|
118
|
-
if isinstance(dataset, jax.Array):
|
|
120
|
+
if isinstance(dataset, (jax.Array, np.ndarray)):
|
|
119
121
|
return _MapFetcher(dataset, collate_fn)
|
|
120
122
|
|
|
121
123
|
return _create_fetcher(dataset, collate_fn)
|
|
@@ -4,10 +4,9 @@ from collections.abc import Iterator, Sequence
|
|
|
4
4
|
from itertools import islice
|
|
5
5
|
from typing import TYPE_CHECKING, TypeVar
|
|
6
6
|
|
|
7
|
-
import jax
|
|
8
|
-
import jax.numpy as jnp
|
|
9
7
|
import numpy as np
|
|
10
8
|
|
|
9
|
+
from .. import random as random_
|
|
11
10
|
from . import datasets as datasets_
|
|
12
11
|
|
|
13
12
|
if TYPE_CHECKING:
|
|
@@ -33,24 +32,10 @@ class KFold:
|
|
|
33
32
|
self.n_splits = n_splits
|
|
34
33
|
self.shuffle = shuffle
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
raw = jnp.zeros((), dtype=jnp.uint32)
|
|
41
|
-
|
|
42
|
-
if engine.is_global_zero:
|
|
43
|
-
# Extract a valid integer representation from the nnx PRNGKey
|
|
44
|
-
key = engine.rngs.default()
|
|
45
|
-
raw = jax.random.key_data(key)[0]
|
|
46
|
-
|
|
47
|
-
# Broadcast the array and extract the integer scalar
|
|
48
|
-
broadcasted_seed = engine.broadcast(raw, src=0)
|
|
49
|
-
self.seed = int(broadcasted_seed)
|
|
50
|
-
else:
|
|
51
|
-
self.seed = seed
|
|
52
|
-
else:
|
|
53
|
-
self.seed = seed if seed is not None else 42
|
|
35
|
+
if seed is None and engine is not None and shuffle:
|
|
36
|
+
# The engine's ``data`` stream is identical on all processes, so no syncing is needed
|
|
37
|
+
seed = random_.key_to_seed(engine.rngs.data())
|
|
38
|
+
self.seed = seed if seed is not None else 42
|
|
54
39
|
|
|
55
40
|
# -- public API ----------------------------------------------------------
|
|
56
41
|
|
|
@@ -3,13 +3,12 @@ import contextlib
|
|
|
3
3
|
import functools
|
|
4
4
|
import itertools
|
|
5
5
|
import math
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Final, Generic, Never, TypeVar
|
|
7
7
|
|
|
8
8
|
import beartype
|
|
9
9
|
import jax
|
|
10
10
|
import jaxtyping as jt
|
|
11
11
|
import numpy as np
|
|
12
|
-
from typing_extensions import Never
|
|
13
12
|
|
|
14
13
|
from . import _types
|
|
15
14
|
|
|
@@ -67,16 +66,33 @@ class SequentialSampler(_types.Sampler[int]):
|
|
|
67
66
|
|
|
68
67
|
|
|
69
68
|
class RandomSampler(_types.Sampler[int]):
|
|
70
|
-
"""Randomly sample integer index up to a given ``length`` with possible ``replacements``.
|
|
69
|
+
"""Randomly sample integer index up to a given ``length`` with possible ``replacements``.
|
|
70
|
+
|
|
71
|
+
If a ``seed`` is set, each iteration (epoch) draws a different, but reproducible, order from
|
|
72
|
+
``(seed, epoch)``. When a loader is set up by a :class:`reax.Engine`, any unseeded
|
|
73
|
+
``RandomSampler`` is given a seed drawn from the engine's ``data`` stream, so shuffling follows
|
|
74
|
+
the engine seed and is identical on all processes (as needed for distributed sharding). If
|
|
75
|
+
no seed is set, the global numpy generator is used.
|
|
76
|
+
"""
|
|
71
77
|
|
|
72
78
|
SAMPLE_SIZE = 32 # Used to control the number of samples we generate internally at once
|
|
73
79
|
|
|
74
|
-
def __init__(
|
|
80
|
+
def __init__(
|
|
81
|
+
self,
|
|
82
|
+
length: int,
|
|
83
|
+
replacements: bool = False,
|
|
84
|
+
num_samples: int | None = None,
|
|
85
|
+
seed: int | None = None,
|
|
86
|
+
):
|
|
75
87
|
# Params
|
|
76
88
|
self._length: Final[int] = length
|
|
77
89
|
self._replacements: Final[bool] = replacements
|
|
78
90
|
self._num_samples: Final[int | None] = num_samples
|
|
79
91
|
|
|
92
|
+
# State
|
|
93
|
+
self.seed: int | None = seed
|
|
94
|
+
self._epoch: int = 0
|
|
95
|
+
|
|
80
96
|
@property
|
|
81
97
|
def num_samples(self) -> int:
|
|
82
98
|
"""Num samples."""
|
|
@@ -85,26 +101,53 @@ class RandomSampler(_types.Sampler[int]):
|
|
|
85
101
|
|
|
86
102
|
return self._num_samples # Fixed number of samples
|
|
87
103
|
|
|
104
|
+
@property
|
|
105
|
+
def epoch(self) -> int:
|
|
106
|
+
"""The number of the next iteration, used together with :attr:`seed` to set the order."""
|
|
107
|
+
return self._epoch
|
|
108
|
+
|
|
109
|
+
def set_epoch(self, epoch: int) -> None:
|
|
110
|
+
"""Set the epoch used for the next iteration (e.g. when resuming)."""
|
|
111
|
+
self._epoch = epoch
|
|
112
|
+
|
|
88
113
|
def __len__(self) -> int:
|
|
89
114
|
"""Len function."""
|
|
90
115
|
return self.num_samples
|
|
91
116
|
|
|
92
117
|
def __iter__(self) -> Iterator[int]:
|
|
93
118
|
"""Iter function."""
|
|
119
|
+
if self.seed is None:
|
|
120
|
+
# Legacy behaviour: use the global numpy generator
|
|
121
|
+
integers, permutation = np.random.randint, np.random.permutation
|
|
122
|
+
else:
|
|
123
|
+
rng = np.random.default_rng([self.seed, self._epoch])
|
|
124
|
+
integers, permutation = rng.integers, rng.permutation
|
|
125
|
+
self._epoch += 1
|
|
126
|
+
|
|
94
127
|
total = self._length
|
|
95
128
|
|
|
96
129
|
if self._replacements:
|
|
97
130
|
for _ in range(self.num_samples // self.SAMPLE_SIZE):
|
|
98
|
-
yield from map(int,
|
|
131
|
+
yield from map(int, integers(0, total, size=(self.SAMPLE_SIZE,)))
|
|
99
132
|
|
|
100
|
-
yield from map(
|
|
101
|
-
int, np.random.randint(0, high=total, size=(self.num_samples % self.SAMPLE_SIZE,))
|
|
102
|
-
)
|
|
133
|
+
yield from map(int, integers(0, total, size=(self.num_samples % self.SAMPLE_SIZE,)))
|
|
103
134
|
else:
|
|
104
135
|
for _ in range(self.num_samples // total):
|
|
105
|
-
yield from map(int,
|
|
136
|
+
yield from map(int, permutation(total))
|
|
106
137
|
|
|
107
|
-
yield from map(int,
|
|
138
|
+
yield from map(int, permutation(total)[: self.num_samples % total])
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def iter_samplers(sampler: Any) -> Iterator[Any]:
|
|
142
|
+
"""Iterate over ``sampler`` and every sampler it (transitively) wraps."""
|
|
143
|
+
while sampler is not None:
|
|
144
|
+
yield sampler
|
|
145
|
+
if isinstance(sampler, ChunkedSampler):
|
|
146
|
+
sampler = sampler.sampler
|
|
147
|
+
elif isinstance(sampler, DistributedSampler):
|
|
148
|
+
sampler = sampler.inner
|
|
149
|
+
else:
|
|
150
|
+
return
|
|
108
151
|
|
|
109
152
|
|
|
110
153
|
class ChunkedSampler(Generic[_IdxT], _types.Sampler[list[_IdxT]]):
|
|
@@ -41,7 +41,7 @@ class BoringModel(modules.Module):
|
|
|
41
41
|
def configure_model(self, _stage, batch: Any, /) -> None:
|
|
42
42
|
"""Configure our model."""
|
|
43
43
|
if self._parameters is None:
|
|
44
|
-
params = self.layer.init(self.rngs(), batch[0])
|
|
44
|
+
params = self.layer.init(self.rngs.params(), batch[0])
|
|
45
45
|
self.set_parameters(params)
|
|
46
46
|
|
|
47
47
|
def forward(self, x: jax.Array) -> jax.Array:
|
|
@@ -78,10 +78,12 @@ class EarlyStopping(hooks.TrainerListener):
|
|
|
78
78
|
|
|
79
79
|
.. note::
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
The patience parameter counts the number of validation checks with no
|
|
82
|
+
improvement, and not the number of training epochs. Therefore, with parameters
|
|
83
|
+
``check_val_every_n_epoch=10`` and ``patience=3``, the trainer will perform at
|
|
84
|
+
least 40 training epochs before being stopped.
|
|
85
|
+
|
|
86
|
+
Defaults to 3.
|
|
85
87
|
verbose (bool, optional): Verbosity mode, defaults to False.
|
|
86
88
|
mode (Literal["min", "max"], optional): One of ``'min'``,
|
|
87
89
|
``'max'``. In ``'min'`` mode, training will stop when the
|