reax 0.7.0.dev5__tar.gz → 0.7.0.dev6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/PKG-INFO +5 -4
  2. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/pyproject.toml +6 -5
  3. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/__init__.py +1 -2
  4. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_engine.py +62 -11
  5. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_module_hooks.py +4 -4
  6. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_datasources.py +2 -17
  7. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/datasets.py +15 -11
  8. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/fetchers.py +4 -2
  9. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/kfold.py +5 -20
  10. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/samplers.py +53 -10
  11. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/boring_classes.py +1 -1
  12. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/early_stopping.py +6 -4
  13. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/model_checkpoint.py +22 -22
  14. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/progress_bar.py +2 -4
  15. reax-0.7.0.dev6/src/reax/loggers/__init__.py +14 -0
  16. reax-0.7.0.dev6/src/reax/loggers/_figures.py +219 -0
  17. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/logger.py +131 -2
  18. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/mlflow.py +153 -16
  19. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/tensorboard.py +17 -1
  20. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_classification.py +19 -23
  21. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_metric.py +2 -2
  22. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/aggregation.py +2 -2
  23. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/modules.py +47 -5
  24. reax-0.7.0.dev6/src/reax/random.py +156 -0
  25. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_checkpointing.py +5 -0
  26. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/trainer.py +22 -4
  27. reax-0.7.0.dev5/src/reax/loggers/__init__.py +0 -7
  28. reax-0.7.0.dev5/src/reax/random.py +0 -22
  29. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/LICENSE.txt +0 -0
  30. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/README.rst +0 -0
  31. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_vendor/__init__.py +0 -0
  32. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/_vendor/simple_pytree.py +0 -0
  33. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/__init__.py +0 -0
  34. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_datasource_manager.py +0 -0
  35. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_loaders.py +0 -0
  36. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/_types.py +0 -0
  37. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/collate.py +0 -0
  38. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/dataloaders.py +0 -0
  39. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/datamodules.py +0 -0
  40. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/data/utils.py +0 -0
  41. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/__init__.py +0 -0
  42. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/demos/mnist.py +0 -0
  43. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/exceptions.py +0 -0
  44. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/hooks.py +0 -0
  45. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/keys.py +0 -0
  46. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/LICENSE +0 -0
  47. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/__init__.py +0 -0
  48. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/lightning/rank_zero.py +0 -0
  49. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/__init__.py +0 -0
  50. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/checkpointer.py +0 -0
  51. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/__init__.py +0 -0
  52. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/rich_progress.py +0 -0
  53. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/progress/tqdm_progress.py +0 -0
  54. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/listeners/utils.py +0 -0
  55. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/_utils.py +0 -0
  56. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/csv_logs.py +0 -0
  57. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/loggers/pandas.py +0 -0
  58. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/__init__.py +0 -0
  59. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_evaluators.py +0 -0
  60. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_globals.py +0 -0
  61. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/_registry.py +0 -0
  62. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/collections.py +0 -0
  63. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/__init__.py +0 -0
  64. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/types.py +0 -0
  65. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/jm/utils.py +0 -0
  66. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/regression.py +0 -0
  67. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/metrics/utils.py +0 -0
  68. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/optimizers.py +0 -0
  69. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/plugins.py +0 -0
  70. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/__init__.py +0 -0
  71. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/jax_profiler.py +0 -0
  72. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/profilers/profiler.py +0 -0
  73. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/pytree.py +0 -0
  74. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/results.py +0 -0
  75. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/saving.py +0 -0
  76. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/__init__.py +0 -0
  77. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/_loops.py +0 -0
  78. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/_timer.py +0 -0
  79. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/common.py +0 -0
  80. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/fit.py +0 -0
  81. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/predict.py +0 -0
  82. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/stages.py +0 -0
  83. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/stats.py +0 -0
  84. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/test.py +0 -0
  85. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/train.py +0 -0
  86. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/stages/validation.py +0 -0
  87. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/__init__.py +0 -0
  88. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_factory.py +0 -0
  89. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_jax.py +0 -0
  90. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_parallel.py +0 -0
  91. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_single_device.py +0 -0
  92. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_strategies.py +0 -0
  93. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/strategies/_utils.py +0 -0
  94. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/testing.py +0 -0
  95. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/__init__.py +0 -0
  96. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_checkpoint_connector.py +0 -0
  97. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_hooks_map.py +0 -0
  98. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_logger_connector.py +0 -0
  99. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/training/_utils.py +0 -0
  100. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/types.py +0 -0
  101. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/__init__.py +0 -0
  102. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/arrays.py +0 -0
  103. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/containers.py +0 -0
  104. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/events.py +0 -0
  105. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/grads.py +0 -0
  106. {reax-0.7.0.dev5 → reax-0.7.0.dev6}/src/reax/utils/masking.py +0 -0
@@ -1,15 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: reax
3
- Version: 0.7.0.dev5
3
+ Version: 0.7.0.dev6
4
4
  Summary: REAX: A simple training framework for JAX-based projects
5
5
  Keywords: machine learning,jax,research
6
6
  Author-email: Martin Uhrin <martin.uhrin.10@ucl.ac.uk>
7
- Requires-Python: >=3.10
7
+ Requires-Python: >=3.11
8
8
  Description-Content-Type: text/x-rst
9
9
  Classifier: Development Status :: 4 - Beta
10
10
  Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
11
11
  Classifier: Programming Language :: Python
12
- Classifier: Programming Language :: Python :: 3.10
13
12
  Classifier: Programming Language :: Python :: 3.11
14
13
  Classifier: Programming Language :: Python :: 3.12
15
14
  Classifier: Programming Language :: Python :: 3.13
@@ -32,9 +31,11 @@ Requires-Dist: typing-extensions
32
31
  Requires-Dist: cloudpickle ; extra == "dev"
33
32
  Requires-Dist: flit ; extra == "dev"
34
33
  Requires-Dist: ipython ; extra == "dev"
35
- Requires-Dist: mlflow<3.7 ; extra == "dev"
34
+ Requires-Dist: matplotlib ; extra == "dev"
35
+ Requires-Dist: mlflow>=3.15 ; extra == "dev"
36
36
  Requires-Dist: omegaconf ; extra == "dev"
37
37
  Requires-Dist: pandas ; extra == "dev"
38
+ Requires-Dist: pillow ; extra == "dev"
38
39
  Requires-Dist: pytest ; extra == "dev"
39
40
  Requires-Dist: pytest-cov ; extra == "dev"
40
41
  Requires-Dist: pytest-env ; extra == "dev"
@@ -14,14 +14,13 @@ classifiers = [
14
14
  'Development Status :: 4 - Beta',
15
15
  'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
16
16
  'Programming Language :: Python',
17
- 'Programming Language :: Python :: 3.10',
18
17
  'Programming Language :: Python :: 3.11',
19
18
  'Programming Language :: Python :: 3.12',
20
19
  'Programming Language :: Python :: 3.13',
21
20
  'Programming Language :: Python :: 3.14',
22
21
  ]
23
22
  keywords = ['machine learning', 'jax', 'research']
24
- requires-python = '>=3.10'
23
+ requires-python = '>=3.11'
25
24
  dependencies = [
26
25
  'beartype',
27
26
  "einops",
@@ -59,9 +58,11 @@ dev = [
59
58
  'cloudpickle',
60
59
  'flit',
61
60
  'ipython',
62
- "mlflow<3.7",
61
+ 'matplotlib',
62
+ "mlflow>=3.15",
63
63
  "omegaconf",
64
64
  'pandas',
65
+ 'pillow',
65
66
  'pytest',
66
67
  'pytest-cov',
67
68
  'pytest-env',
@@ -101,7 +102,7 @@ mae = "reax.metrics:MeanAbsoluteError"
101
102
 
102
103
  [tool.ruff]
103
104
  line-length = 100
104
- target-version = "py310"
105
+ target-version = "py311"
105
106
  exclude = [
106
107
  'migrations',
107
108
  ]
@@ -280,7 +281,7 @@ envlist = py311
280
281
  [testenv]
281
282
  usedevelop = true
282
283
 
283
- [testenv:py{310,311,312,313}]
284
+ [testenv:py{311,312,313,314}]
284
285
  description = Run the unit tests
285
286
  extras =
286
287
  dev
@@ -42,7 +42,6 @@ __all__ = (
42
42
  + saving.__all__
43
43
  + strategies.__all__
44
44
  + training.__all__
45
- + data.__all__
46
45
  # Modules
47
46
  + (
48
47
  "data",
@@ -68,4 +67,4 @@ __all__ = (
68
67
  )
69
68
  )
70
69
 
71
- __version__ = "0.7.0.dev5"
70
+ __version__ = "0.7.0.dev6"
@@ -10,11 +10,13 @@ from flax import nnx
10
10
  import fsspec.utils
11
11
  import jax
12
12
  import jaxtyping as jt
13
+ import numpy as np
13
14
 
14
15
  from . import data as data_
15
16
  from . import hooks, modules, optimizers, profilers, strategies
16
17
  from . import loggers as loggers_
17
18
  from . import metrics as metrics_
19
+ from . import random as random_
18
20
  from .data import DeviceDataLoader
19
21
  from .lightning import rank_zero
20
22
  from .training import _logger_connector
@@ -43,10 +45,6 @@ class Engine:
43
45
  It encapsulates the environment setup defined by the chosen strategy,
44
46
  managing details like accelerator configuration, process launching (e.g.,
45
47
  via MPI or Python's multiprocessing), and backend initialization.
46
-
47
- Attributes:
48
- strategy (object): The execution strategy defining the environment and distributed
49
- communication.
50
48
  """
51
49
 
52
50
  # pylint: disable=too-many-public-methods
@@ -65,6 +63,7 @@ class Engine:
65
63
  default_root_dir: "reax.types.Path | None" = None,
66
64
  profiler: "reax.Profiler | str | None" = None,
67
65
  metric_evaluator: "reax.metrics.MetricEvaluator | None" = None,
66
+ seed: int | None = None,
68
67
  ):
69
68
  """
70
69
  Initializes the Engine with execution parameters and components.
@@ -86,11 +85,18 @@ class Engine:
86
85
  listeners (list[reax.TrainerListener] | reax.TrainerListener | None): A single or list
87
86
  of :class:`reax.TrainerListener` objects for hooking into events.
88
87
  deterministic (bool): Currently not supported, a warning is issued if set.
89
- rngs (nnx.Rngs): The initial random number generator state used for all training steps.
90
- Defaults to a new :class:`nnx.Rngs(0)`.
88
+ rngs (nnx.Rngs): Advanced: an explicit random number generator state to use instead of
89
+ the one REAX builds from ``seed``. Cannot be combined with ``seed``.
91
90
  default_root_dir (typing.Path | None): The default directory path for saving logs and
92
91
  checkpoints if not specified elsewhere. Defaults to the current working directory.
93
- """
92
+ seed (int | None): The single seed from which all randomness is derived (see
93
+ :mod:`reax.random`). Pass an integer for a reproducible run. If ``None``, a seed
94
+ is drawn at random (and shared across all processes), so every run is different.
95
+ Either way, the seed is logged, stored in checkpoints and available as
96
+ :attr:`seed`, so any run can be reproduced.
97
+ """
98
+ if rngs is not None and seed is not None:
99
+ raise ValueError("Pass either `seed` or `rngs`, not both.")
94
100
  if deterministic:
95
101
  _LOGGER.warning("`deterministic=True` is not supported yet, ignoring.")
96
102
  if precision is not None:
@@ -116,8 +122,17 @@ class Engine:
116
122
  self._events = events.EventGenerator[hooks.TrainerListener](
117
123
  default_args=(weakref.proxy(self),)
118
124
  )
119
- self._rngs = rngs if rngs is not None else nnx.Rngs(0)
120
125
  self._loggers: list[loggers_.Logger] = _init_loggers(logger, self.default_root_dir)
126
+ self._seed: int | None
127
+ if rngs is not None:
128
+ self._seed = None
129
+ self._rngs = rngs
130
+ else:
131
+ self._seed = self._resolve_seed(seed)
132
+ self._rngs = random_.create_rngs(self._seed, process_index=self.process_index)
133
+ random_.seed_globals(self._seed)
134
+ if self.is_global_zero:
135
+ _LOGGER.info("Using seed: %i", self._seed)
121
136
  self._logging = _logger_connector.TrainerLogging()
122
137
  self._profiler: profilers.Profiler = _init_profiler(profiler)
123
138
 
@@ -192,13 +207,35 @@ class Engine:
192
207
  @property
193
208
  def rngs(self) -> nnx.Rngs:
194
209
  """
195
- The current random number generator state used by the Engine.
210
+ The random number generators from which all randomness should be drawn.
211
+
212
+ These have independent named streams (see :mod:`reax.random`): ``params``, ``data`` and
213
+ ``default`` are identical on all processes, while ``dropout`` differs between processes.
196
214
 
197
215
  Returns:
198
216
  nnx.Rngs: The RNG state object.
199
217
  """
200
218
  return self._rngs
201
219
 
220
+ @property
221
+ def seed(self) -> int | None:
222
+ """
223
+ The seed from which :attr:`rngs` were created, or ``None`` if explicit ``rngs`` were
224
+ passed in.
225
+ """
226
+ return self._seed
227
+
228
+ def _resolve_seed(self, seed: int | None) -> int:
229
+ """Get the seed to use, drawing a random one (shared by all processes) if needed."""
230
+ if seed is not None:
231
+ return int(seed)
232
+
233
+ seed = random_.draw_seed()
234
+ if self.process_count > 1:
235
+ seed = int(self.broadcast(np.asarray(seed, dtype=np.int32), src=0))
236
+
237
+ return seed
238
+
202
239
  @property
203
240
  def strategy(self) -> strategies.Strategy:
204
241
  """
@@ -339,7 +376,8 @@ class Engine:
339
376
  reax.data.DeviceDataLoader | list[reax.data.DeviceDataLoader]: The prepared
340
377
  dataloader(s), wrapped as a :class:`~reax.data.DeviceDataLoader`.
341
378
  """
342
- loaders = list(map(self.strategy.setup_dataloader, args))
379
+ loaders = list(map(self._seed_loader, args))
380
+ loaders = list(map(self.strategy.setup_dataloader, loaders))
343
381
  # Wrap in DeviceLoader so all data is already on the selected device
344
382
  loaders = list(map(self._wrap_loader, loaders))
345
383
  if len(args) == 1:
@@ -436,6 +474,19 @@ class Engine:
436
474
  """
437
475
  return self._strategy.compute(metric)
438
476
 
477
+ def _seed_loader(self, loader: "reax.DataLoader") -> "reax.DataLoader":
478
+ """Give any unseeded :class:`~reax.data.samplers.RandomSampler` used by the loader a seed
479
+ drawn from the ``data`` stream, so shuffling follows the engine seed and is the same on
480
+ all processes."""
481
+ inner = loader
482
+ while isinstance(inner, DeviceDataLoader):
483
+ inner = inner.parent
484
+ for sub in data_.samplers.iter_samplers(getattr(inner, "sampler", None)):
485
+ if isinstance(sub, data_.samplers.RandomSampler) and sub.seed is None:
486
+ sub.seed = random_.key_to_seed(self._rngs.data())
487
+
488
+ return loader
489
+
439
490
  def _wrap_loader(self, loader: "reax.DataLoader") -> "reax.data.DeviceDataLoader":
440
491
  """
441
492
  Internal utility to wrap a standard DataLoader into a DeviceDataLoader.
@@ -465,7 +516,7 @@ def _(arg: modules.Module, engine: Engine) -> modules.Module:
465
516
 
466
517
  @_setup.register
467
518
  def _(arg: data_.DataLoader, engine: Engine) -> data_.DataLoader:
468
- return engine.strategy.setup_dataloader(arg)
519
+ return engine.strategy.setup_dataloader(engine._seed_loader(arg)) # pylint: disable=protected-access
469
520
 
470
521
 
471
522
  @jt.jaxtyped(typechecker=beartype.beartype)
@@ -300,10 +300,10 @@ class ModuleHooks:
300
300
 
301
301
  Only called when using automatic optimization (see
302
302
  :attr:`~reax.Module.automatic_optimization`). This is called for every training batch,
303
- including when gradients are being accumulated (see
304
- :paramref:`~reax.Trainer.fit.accumulate_grad_batches`), as accumulation is performed by
305
- the optimizer. Any transformations that are part of the optimizer (e.g. gradient
306
- clipping) will not have been applied yet.
303
+ including when gradients are being accumulated (see the
304
+ ``accumulate_grad_batches`` argument of :meth:`~reax.Trainer.fit`), as accumulation is
305
+ performed by the optimizer. Any transformations that are part of the optimizer (e.g.
306
+ gradient clipping) will not have been applied yet.
307
307
 
308
308
  Args:
309
309
  optimizer: The optimizer that is about to be updated.
@@ -52,7 +52,7 @@ class DataSource(Generic[_T_co, U]):
52
52
  self.some_state = some_other_state()
53
53
 
54
54
  In a distributed environment, ``prepare_data`` can be called in two ways
55
- (using :ref:`prepare_data_per_node<common/lightning_module:prepare_data_per_node>`)
55
+ (using :attr:`prepare_data_per_node`)
56
56
 
57
57
  1. Once per node. This is the default and is only called on LOCAL_RANK=0.
58
58
  2. Once in total. Only called on GLOBAL_RANK=0.
@@ -135,11 +135,6 @@ class DataSource(Generic[_T_co, U]):
135
135
  def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
136
136
  """An iterable or collection of iterables specifying training samples.
137
137
 
138
- For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
139
-
140
- The dataloader you return will not be reloaded unless you set
141
- :paramref:`~reax.Trainer.fit(reload_dataloaders_every_n_epochs=k)` to a positive integer.
142
-
143
138
  For data processing use the following pattern:
144
139
 
145
140
  - download in :meth:`prepare_data`
@@ -166,12 +161,6 @@ class DataSource(Generic[_T_co, U]):
166
161
  def val_dataloader(self) -> Optional["reax.DataLoader[_T_co, U]"]:
167
162
  r"""An iterable or collection of iterables specifying validation samples.
168
163
 
169
- For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
170
-
171
- The dataloader you return will not be reloaded unless you set
172
- :paramref:`~lightning.pytorch.trainer.trainer.Trainer.reload_dataloaders_every_n_epochs` to
173
- a positive integer.
174
-
175
164
  It's recommended that all data downloads and preparation happen in :meth:`prepare_data`.
176
165
 
177
166
  - :meth:`~reax.Trainer.fit`
@@ -192,8 +181,6 @@ class DataSource(Generic[_T_co, U]):
192
181
  def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
193
182
  r"""An iterable or collection of iterables specifying test samples.
194
183
 
195
- For more information about multiple dataloaders, this :ref:`section <multiple-dataloaders>`.
196
-
197
184
  For data processing use the following pattern:
198
185
 
199
186
  - download in :meth:`prepare_data`
@@ -204,7 +191,7 @@ class DataSource(Generic[_T_co, U]):
204
191
  Warning:
205
192
  do not assign state in prepare_data
206
193
 
207
- - :meth:`~lightning.pytorch.trainer.trainer.Trainer.test`
194
+ - :meth:`~reax.Trainer.test`
208
195
  - :meth:`prepare_data`
209
196
  - :meth:`setup`
210
197
 
@@ -224,8 +211,6 @@ class DataSource(Generic[_T_co, U]):
224
211
  def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
225
212
  r"""An iterable or collection of iterables specifying prediction samples.
226
213
 
227
- For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
228
-
229
214
  It's recommended that all data downloads and preparation happen in :meth:`prepare_data`.
230
215
 
231
216
  - :meth:`~reax.Trainer.predict`
@@ -2,17 +2,20 @@ import bisect
2
2
  from collections.abc import Iterable, Sequence
3
3
  import itertools
4
4
  import math
5
- from typing import TypeVar, cast
5
+ from typing import TYPE_CHECKING, TypeVar, cast
6
6
  import warnings
7
7
 
8
8
  import beartype
9
- from flax import nnx
10
9
  import jax.random
11
10
  import jaxtyping as jt
12
11
  import numpy as np
13
12
 
13
+ from .. import random as random_
14
14
  from . import _types
15
15
 
16
+ if TYPE_CHECKING:
17
+ from flax import nnx
18
+
16
19
  __all__ = ("ArrayDataset", "ConcatDataset", "Subset", "random_split")
17
20
 
18
21
  _T = TypeVar("_T")
@@ -132,7 +135,7 @@ class Subset(Sequence[_T_co]):
132
135
 
133
136
 
134
137
  def random_split(
135
- rngs: nnx.Rngs,
138
+ rngs: "nnx.Rngs | nnx.RngStream | jax.Array | int",
136
139
  dataset: Sequence[_T],
137
140
  lengths: Sequence[int | float],
138
141
  ) -> list[Subset[_T]]:
@@ -146,19 +149,19 @@ def random_split(
146
149
  distributed in round-robin fashion to the lengths
147
150
  until there are no remainders left.
148
151
 
149
- Optionally fix the generator for reproducible results, e.g.:
152
+ The split is drawn from ``rngs``; pass ``engine.rngs`` to have it follow the engine seed (and
153
+ be identical on all processes).
150
154
 
151
155
  Example:
152
- >>> # xdoctest: +SKIP
153
- >>> key1 = torch.Generator().manual_seed(42)
154
- >>> key2 = torch.Generator().manual_seed(42)
155
- >>> random_split(key1, range(10), [3, 7])
156
- >>> random_split(key2, range(30), [0.3, 0.3, 0.4])
156
+ >>> rngs = nnx.Rngs(data=42)
157
+ >>> train, val = random_split(rngs, range(10), [3, 7])
158
+ >>> train, val, test = random_split(jax.random.key(0), range(30), [0.3, 0.3, 0.4])
157
159
 
158
160
  Args:
161
+ rngs: The source of randomness: an :class:`flax.nnx.Rngs` (a key is drawn from its
162
+ ``data`` stream), an :class:`flax.nnx.RngStream`, a JAX key or an integer seed.
159
163
  dataset (Dataset): Dataset to be split
160
164
  lengths (sequence): lengths or fractions of splits to be produced
161
- generator (Generator): Generator used for the random permutation.
162
165
  """
163
166
  if math.isclose(sum(lengths), 1) and sum(lengths) <= 1:
164
167
  subset_lengths: list[int] = []
@@ -184,7 +187,8 @@ def random_split(
184
187
  if sum(lengths) != len(dataset):
185
188
  raise ValueError("Sum of input lengths does not equal the length of the input dataset!")
186
189
 
187
- indices = jax.random.permutation(rngs(), sum(lengths)).tolist()
190
+ key = random_.to_key(rngs, stream="data")
191
+ indices = jax.random.permutation(key, sum(lengths)).tolist()
188
192
  lengths = cast(Sequence[int], lengths)
189
193
  return [
190
194
  Subset(dataset, indices[offset - length : offset])
@@ -5,6 +5,7 @@ import functools
5
5
  from typing import Generic, TypeVar
6
6
 
7
7
  import jax
8
+ import numpy as np
8
9
 
9
10
  from . import _types
10
11
 
@@ -111,11 +112,12 @@ def create_fetcher(
111
112
  CPython's singledispatch C3 linearization places the ``Iterable`` abstract base class before
112
113
  the concrete ``jax.Array`` in the composed MRO. Registering a ``jax.Array`` handler therefore
113
114
  would never be selected and the array would be handled by the iterable fetcher, which ignores
114
- the sampler indices and yields the dataset sequentially. We disambiguate with an explicit
115
+ the sampler indices and yields the dataset sequentially (``numpy.ndarray`` isn't a ``Sequence``
116
+ either, so has the same problem). We disambiguate with an explicit
115
117
  ``isinstance`` check (mirroring the equivalent special case in ``create_sampler``) so the array
116
118
  is treated as a ``_MapFetcher`` and the sampler indices are actually used.
117
119
  """
118
- if isinstance(dataset, jax.Array):
120
+ if isinstance(dataset, (jax.Array, np.ndarray)):
119
121
  return _MapFetcher(dataset, collate_fn)
120
122
 
121
123
  return _create_fetcher(dataset, collate_fn)
@@ -4,10 +4,9 @@ from collections.abc import Iterator, Sequence
4
4
  from itertools import islice
5
5
  from typing import TYPE_CHECKING, TypeVar
6
6
 
7
- import jax
8
- import jax.numpy as jnp
9
7
  import numpy as np
10
8
 
9
+ from .. import random as random_
11
10
  from . import datasets as datasets_
12
11
 
13
12
  if TYPE_CHECKING:
@@ -33,24 +32,10 @@ class KFold:
33
32
  self.n_splits = n_splits
34
33
  self.shuffle = shuffle
35
34
 
36
- # Synchronise seed across distributed ranks
37
- if engine is not None and shuffle:
38
- if seed is None:
39
- # Initialize matching array structures on all ranks
40
- raw = jnp.zeros((), dtype=jnp.uint32)
41
-
42
- if engine.is_global_zero:
43
- # Extract a valid integer representation from the nnx PRNGKey
44
- key = engine.rngs.default()
45
- raw = jax.random.key_data(key)[0]
46
-
47
- # Broadcast the array and extract the integer scalar
48
- broadcasted_seed = engine.broadcast(raw, src=0)
49
- self.seed = int(broadcasted_seed)
50
- else:
51
- self.seed = seed
52
- else:
53
- self.seed = seed if seed is not None else 42
35
+ if seed is None and engine is not None and shuffle:
36
+ # The engine's ``data`` stream is identical on all processes, so no syncing is needed
37
+ seed = random_.key_to_seed(engine.rngs.data())
38
+ self.seed = seed if seed is not None else 42
54
39
 
55
40
  # -- public API ----------------------------------------------------------
56
41
 
@@ -3,13 +3,12 @@ import contextlib
3
3
  import functools
4
4
  import itertools
5
5
  import math
6
- from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar
6
+ from typing import TYPE_CHECKING, Any, Final, Generic, Never, TypeVar
7
7
 
8
8
  import beartype
9
9
  import jax
10
10
  import jaxtyping as jt
11
11
  import numpy as np
12
- from typing_extensions import Never
13
12
 
14
13
  from . import _types
15
14
 
@@ -67,16 +66,33 @@ class SequentialSampler(_types.Sampler[int]):
67
66
 
68
67
 
69
68
  class RandomSampler(_types.Sampler[int]):
70
- """Randomly sample integer index up to a given ``length`` with possible ``replacements``."""
69
+ """Randomly sample integer index up to a given ``length`` with possible ``replacements``.
70
+
71
+ If a ``seed`` is set, each iteration (epoch) draws a different, but reproducible, order from
72
+ ``(seed, epoch)``. When a loader is set up by a :class:`reax.Engine`, any unseeded
73
+ ``RandomSampler`` is given a seed drawn from the engine's ``data`` stream, so shuffling follows
74
+ the engine seed and is identical on all processes (as needed for distributed sharding). If
75
+ no seed is set, the global numpy generator is used.
76
+ """
71
77
 
72
78
  SAMPLE_SIZE = 32 # Used to control the number of samples we generate internally at once
73
79
 
74
- def __init__(self, length: int, replacements: bool = False, num_samples: int | None = None):
80
+ def __init__(
81
+ self,
82
+ length: int,
83
+ replacements: bool = False,
84
+ num_samples: int | None = None,
85
+ seed: int | None = None,
86
+ ):
75
87
  # Params
76
88
  self._length: Final[int] = length
77
89
  self._replacements: Final[bool] = replacements
78
90
  self._num_samples: Final[int | None] = num_samples
79
91
 
92
+ # State
93
+ self.seed: int | None = seed
94
+ self._epoch: int = 0
95
+
80
96
  @property
81
97
  def num_samples(self) -> int:
82
98
  """Num samples."""
@@ -85,26 +101,53 @@ class RandomSampler(_types.Sampler[int]):
85
101
 
86
102
  return self._num_samples # Fixed number of samples
87
103
 
104
+ @property
105
+ def epoch(self) -> int:
106
+ """The number of the next iteration, used together with :attr:`seed` to set the order."""
107
+ return self._epoch
108
+
109
+ def set_epoch(self, epoch: int) -> None:
110
+ """Set the epoch used for the next iteration (e.g. when resuming)."""
111
+ self._epoch = epoch
112
+
88
113
  def __len__(self) -> int:
89
114
  """Len function."""
90
115
  return self.num_samples
91
116
 
92
117
  def __iter__(self) -> Iterator[int]:
93
118
  """Iter function."""
119
+ if self.seed is None:
120
+ # Legacy behaviour: use the global numpy generator
121
+ integers, permutation = np.random.randint, np.random.permutation
122
+ else:
123
+ rng = np.random.default_rng([self.seed, self._epoch])
124
+ integers, permutation = rng.integers, rng.permutation
125
+ self._epoch += 1
126
+
94
127
  total = self._length
95
128
 
96
129
  if self._replacements:
97
130
  for _ in range(self.num_samples // self.SAMPLE_SIZE):
98
- yield from map(int, np.random.randint(0, high=total, size=(self.SAMPLE_SIZE,)))
131
+ yield from map(int, integers(0, total, size=(self.SAMPLE_SIZE,)))
99
132
 
100
- yield from map(
101
- int, np.random.randint(0, high=total, size=(self.num_samples % self.SAMPLE_SIZE,))
102
- )
133
+ yield from map(int, integers(0, total, size=(self.num_samples % self.SAMPLE_SIZE,)))
103
134
  else:
104
135
  for _ in range(self.num_samples // total):
105
- yield from map(int, np.random.permutation(total))
136
+ yield from map(int, permutation(total))
106
137
 
107
- yield from map(int, np.random.permutation(total)[: self.num_samples % total])
138
+ yield from map(int, permutation(total)[: self.num_samples % total])
139
+
140
+
141
+ def iter_samplers(sampler: Any) -> Iterator[Any]:
142
+ """Iterate over ``sampler`` and every sampler it (transitively) wraps."""
143
+ while sampler is not None:
144
+ yield sampler
145
+ if isinstance(sampler, ChunkedSampler):
146
+ sampler = sampler.sampler
147
+ elif isinstance(sampler, DistributedSampler):
148
+ sampler = sampler.inner
149
+ else:
150
+ return
108
151
 
109
152
 
110
153
  class ChunkedSampler(Generic[_IdxT], _types.Sampler[list[_IdxT]]):
@@ -41,7 +41,7 @@ class BoringModel(modules.Module):
41
41
  def configure_model(self, _stage, batch: Any, /) -> None:
42
42
  """Configure our model."""
43
43
  if self._parameters is None:
44
- params = self.layer.init(self.rngs(), batch[0])
44
+ params = self.layer.init(self.rngs.params(), batch[0])
45
45
  self.set_parameters(params)
46
46
 
47
47
  def forward(self, x: jax.Array) -> jax.Array:
@@ -78,10 +78,12 @@ class EarlyStopping(hooks.TrainerListener):
78
78
 
79
79
  .. note::
80
80
 
81
- It must be noted that the patience parameter counts the number of validation checks with
82
- no improvement, and not the number of training epochs. Therefore, with parameters
83
- ``check_val_every_n_epoch=10`` and ``patience=3``, the trainer will perform at least 40
84
- training epochs before being stopped, defaults to 3.
81
+ The patience parameter counts the number of validation checks with no
82
+ improvement, and not the number of training epochs. Therefore, with parameters
83
+ ``check_val_every_n_epoch=10`` and ``patience=3``, the trainer will perform at
84
+ least 40 training epochs before being stopped.
85
+
86
+ Defaults to 3.
85
87
  verbose (bool, optional): Verbosity mode, defaults to False.
86
88
  mode (Literal["min", "max"], optional): One of ``'min'``,
87
89
  ``'max'``. In ``'min'`` mode, training will stop when the