reax 0.6.13.dev1__tar.gz → 0.6.14.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/PKG-INFO +1 -1
  2. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/__init__.py +1 -1
  3. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/_engine.py +1 -1
  4. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_datasources.py +6 -5
  5. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_loaders.py +1 -1
  6. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_types.py +7 -0
  7. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/datamodules.py +11 -8
  8. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/utils.py +1 -1
  9. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/modules.py +19 -3
  10. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/plugins.py +2 -2
  11. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/results.py +66 -12
  12. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/common.py +3 -3
  13. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/fit.py +10 -6
  14. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/predict.py +10 -8
  15. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/stages.py +5 -2
  16. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/test.py +7 -5
  17. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/train.py +22 -19
  18. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/validation.py +10 -7
  19. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_jax.py +2 -2
  20. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_single_device.py +1 -1
  21. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_strategies.py +1 -1
  22. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/trainer.py +2 -0
  23. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/types.py +5 -1
  24. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/LICENSE.txt +0 -0
  25. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/README.rst +0 -0
  26. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/pyproject.toml +0 -0
  27. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/_module_hooks.py +0 -0
  28. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/__init__.py +0 -0
  29. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_datasource_manager.py +0 -0
  30. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/collate.py +0 -0
  31. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/dataloaders.py +0 -0
  32. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/datasets.py +0 -0
  33. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/fetchers.py +0 -0
  34. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/kfold.py +0 -0
  35. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/samplers.py +0 -0
  36. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/__init__.py +0 -0
  37. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/boring_classes.py +0 -0
  38. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/mnist.py +0 -0
  39. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/exceptions.py +0 -0
  40. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/hooks.py +0 -0
  41. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/keys.py +0 -0
  42. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/LICENSE +0 -0
  43. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/__init__.py +0 -0
  44. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/rank_zero.py +0 -0
  45. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/__init__.py +0 -0
  46. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/checkpointer.py +0 -0
  47. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/early_stopping.py +0 -0
  48. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/model_checkpoint.py +0 -0
  49. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/__init__.py +0 -0
  50. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/progress_bar.py +0 -0
  51. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/rich_progress.py +0 -0
  52. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/tqdm_progress.py +0 -0
  53. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/utils.py +0 -0
  54. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/__init__.py +0 -0
  55. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/_utils.py +0 -0
  56. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/csv_logs.py +0 -0
  57. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/logger.py +0 -0
  58. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/mlflow.py +0 -0
  59. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/pandas.py +0 -0
  60. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/tensorboard.py +0 -0
  61. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/__init__.py +0 -0
  62. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_classification.py +0 -0
  63. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_evaluators.py +0 -0
  64. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_globals.py +0 -0
  65. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_metric.py +0 -0
  66. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_registry.py +0 -0
  67. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/aggregation.py +0 -0
  68. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/collections.py +0 -0
  69. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/__init__.py +0 -0
  70. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/types.py +0 -0
  71. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/utils.py +0 -0
  72. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/regression.py +0 -0
  73. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/utils.py +0 -0
  74. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/optimizers.py +0 -0
  75. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/__init__.py +0 -0
  76. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/jax_profiler.py +0 -0
  77. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/profiler.py +0 -0
  78. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/random.py +0 -0
  79. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/saving.py +0 -0
  80. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/__init__.py +0 -0
  81. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/_timer.py +0 -0
  82. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/stats.py +0 -0
  83. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/__init__.py +0 -0
  84. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_factory.py +0 -0
  85. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_parallel.py +0 -0
  86. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_utils.py +0 -0
  87. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/testing.py +0 -0
  88. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/__init__.py +0 -0
  89. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_checkpoint_connector.py +0 -0
  90. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_checkpointing.py +0 -0
  91. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_deprecated.py +0 -0
  92. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_logger_connector.py +0 -0
  93. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/__init__.py +0 -0
  94. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/arrays.py +0 -0
  95. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/containers.py +0 -0
  96. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/events.py +0 -0
  97. {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/grads.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: reax
3
- Version: 0.6.13.dev1
3
+ Version: 0.6.14.dev1
4
4
  Summary: REAX: A simple training framework for JAX-based projects
5
5
  Keywords: machine learning,jax,research
6
6
  Author-email: Martin Uhrin <martin.uhrin.10@ucl.ac.uk>
@@ -68,4 +68,4 @@ __all__ = (
68
68
  )
69
69
  )
70
70
 
71
- __version__ = "0.6.13.dev1"
71
+ __version__ = "0.6.14.dev1"
@@ -415,7 +415,7 @@ class Engine:
415
415
  """
416
416
  self._events.fire_event(getattr(hooks.TrainerListener, name), *args, **kwargs)
417
417
 
418
- def compute(self, metric: "reax.typing.MetricInstance[_OutT]") -> _OutT:
418
+ def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
419
419
  """
420
420
  Computes the final value of a metric across all distributed processes.
421
421
 
@@ -8,9 +8,10 @@ if TYPE_CHECKING:
8
8
  __all__ = ("DataSource",)
9
9
 
10
10
  _T_co = TypeVar("_T_co", covariant=True)
11
+ U = TypeVar("U")
11
12
 
12
13
 
13
- class DataSource(Generic[_T_co]):
14
+ class DataSource(Generic[_T_co, U]):
14
15
  """Interface for classes that provide datasets."""
15
16
 
16
17
  def __init__(self) -> None:
@@ -117,7 +118,7 @@ class DataSource(Generic[_T_co]):
117
118
 
118
119
  # endregion
119
120
 
120
- def train_dataloader(self) -> "reax.DataLoader[_T_co]":
121
+ def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
121
122
  """An iterable or collection of iterables specifying training samples.
122
123
 
123
124
  For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
@@ -148,7 +149,7 @@ class DataSource(Generic[_T_co]):
148
149
  "`train_dataloader` must be implemented to be used with the REAX Trainer"
149
150
  )
150
151
 
151
- def val_dataloader(self) -> Optional["reax.DataLoader[_T_co]"]:
152
+ def val_dataloader(self) -> Optional["reax.DataLoader[_T_co, U]"]:
152
153
  r"""An iterable or collection of iterables specifying validation samples.
153
154
 
154
155
  For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
@@ -174,7 +175,7 @@ class DataSource(Generic[_T_co]):
174
175
 
175
176
  """
176
177
 
177
- def test_dataloader(self) -> "reax.DataLoader[_T_co]":
178
+ def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
178
179
  r"""An iterable or collection of iterables specifying test samples.
179
180
 
180
181
  For more information about multiple dataloaders, this :ref:`section <multiple-dataloaders>`.
@@ -206,7 +207,7 @@ class DataSource(Generic[_T_co]):
206
207
  "`test_dataloader` must be implemented to be used with the REAX Trainer"
207
208
  )
208
209
 
209
- def predict_dataloader(self) -> "reax.DataLoader[_T_co]":
210
+ def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
210
211
  r"""An iterable or collection of iterables specifying prediction samples.
211
212
 
212
213
  For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
@@ -238,5 +238,5 @@ class DeviceDataLoader(_types.DataLoader[_T_co, _U]):
238
238
  return self._loader.sampler
239
239
 
240
240
  @override
241
- def with_new_sampler(self, sampler: "reax.data.Sampler") -> "DataLoader[_T_co, U]":
241
+ def with_new_sampler(self, sampler: "reax.data.Sampler") -> "DeviceDataLoader[_T_co, _U]":
242
242
  return DeviceDataLoader(self._loader.with_new_sampler(sampler), device=self._device)
@@ -17,6 +17,13 @@ CollateFn = Callable[[Sequence[_T_co]], U]
17
17
 
18
18
 
19
19
  class DataLoader(Generic[_T_co, U], Iterable[U], abc.ABC):
20
+ """Abstract base class for loaders that yield processed batches of data.
21
+
22
+ Args:
23
+ _T_co: The type of elements in the underlying dataset.
24
+ U: The type of the batches produced by the loader.
25
+ """
26
+
20
27
  def __len__(self) -> int:
21
28
  return len(self.sampler)
22
29
 
@@ -1,5 +1,5 @@
1
1
  from collections.abc import Iterable
2
- from typing import TYPE_CHECKING, Any
2
+ from typing import TYPE_CHECKING, Any, Generic, TypeVar
3
3
 
4
4
  from flax import nnx
5
5
  from typing_extensions import override
@@ -13,8 +13,11 @@ __all__ = ("DataModule",)
13
13
 
14
14
  Dataset = Any
15
15
 
16
+ _T_co = TypeVar("_T_co", covariant=True)
17
+ U = TypeVar("U")
16
18
 
17
- class DataModule(_datasources.DataSource):
19
+
20
+ class DataModule(Generic[_T_co, U], _datasources.DataSource[_T_co, U]):
18
21
  def __init__(self):
19
22
  super().__init__()
20
23
  self._rngs = nnx.Rngs(0)
@@ -36,14 +39,14 @@ class DataModule(_datasources.DataSource):
36
39
  predict_dataset: Dataset | Iterable[Dataset] | None = None,
37
40
  *,
38
41
  batch_size: int = 1,
39
- ) -> "DataModule":
42
+ ) -> "DataModule[_T_co, U]":
40
43
  """From datasets."""
41
44
  return FromDatasets(
42
45
  train_dataset, val_dataset, test_dataset, predict_dataset, batch_size=batch_size
43
46
  )
44
47
 
45
48
 
46
- class FromDatasets(DataModule):
49
+ class FromDatasets(DataModule[_T_co, U], Generic[_T_co, U]):
47
50
  def __init__(
48
51
  self,
49
52
  train_dataset: Dataset | Iterable[Dataset] | None = None,
@@ -62,21 +65,21 @@ class FromDatasets(DataModule):
62
65
  self._batch_size = batch_size
63
66
 
64
67
  @override
65
- def train_dataloader(self) -> "reax.DataLoader":
68
+ def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
66
69
  """Train dataloader."""
67
70
  return _loaders.ReaxDataLoader(self._train_dataset, batch_size=self._batch_size)
68
71
 
69
72
  @override
70
- def val_dataloader(self) -> "reax.DataLoader":
73
+ def val_dataloader(self) -> "reax.DataLoader[_T_co, U]":
71
74
  """Val dataloader."""
72
75
  return _loaders.ReaxDataLoader(self._val_dataset, batch_size=self._batch_size)
73
76
 
74
77
  @override
75
- def test_dataloader(self) -> "reax.DataLoader":
78
+ def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
76
79
  """Test dataloader."""
77
80
  return _loaders.ReaxDataLoader(self._test_dataset, batch_size=self._batch_size)
78
81
 
79
82
  @override
80
- def predict_dataloader(self) -> "reax.DataLoader":
83
+ def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
81
84
  """Predict dataloader."""
82
85
  return _loaders.ReaxDataLoader(self._predict_dataset, batch_size=self._batch_size)
@@ -22,7 +22,7 @@ class BatchSizer:
22
22
  """Tool for extracting batch sizes from dataset."""
23
23
 
24
24
  def __init__(self):
25
- self._registry = containers.TypeRegistry[Extractor]()
25
+ self._registry: containers.TypeRegistry[Extractor] = containers.TypeRegistry()
26
26
 
27
27
  def register(
28
28
  self,
@@ -1,6 +1,6 @@
1
1
  from collections.abc import Callable, Generator, Sequence
2
2
  import contextlib
3
- from typing import TYPE_CHECKING, Any, Generic, TypedDict, TypeVar
3
+ from typing import TYPE_CHECKING, Any, Generic, TypeAlias, TypedDict, TypeVar
4
4
 
5
5
  import beartype
6
6
  from flax import nnx
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
18
18
 
19
19
  __all__ = ("Module",)
20
20
 
21
- MetricType = "reax.Metric" | jax.typing.ArrayLike
21
+ MetricType: TypeAlias = "reax.Metric | jax.typing.ArrayLike"
22
22
  OutputT_co = TypeVar("OutputT_co", covariant=True)
23
23
  BatchT = TypeVar("BatchT")
24
24
  OptimizerData = tuple[optax.GradientTransformation, Any]
@@ -34,7 +34,7 @@ TrainOutput = LossAndGrad | LossAndGradDict
34
34
 
35
35
 
36
36
  class Module(
37
- Generic[BatchT, OutputT_co], _module_hooks.ModuleHooks, _datasources.DataSource[BatchT]
37
+ Generic[BatchT, OutputT_co], _module_hooks.ModuleHooks, _datasources.DataSource[BatchT, Any]
38
38
  ):
39
39
  example_input_array: BatchT | None
40
40
 
@@ -171,12 +171,27 @@ class Module(
171
171
  logger: bool | None = None,
172
172
  on_step=True,
173
173
  on_epoch=True,
174
+ reduce_fn: "reax.types.ReduceFn" = "mean",
174
175
  ) -> None:
175
176
  """Log a key, value pair.
176
177
 
177
178
  Example::
178
179
 
179
180
  self.log('train_loss', loss)
181
+
182
+ A raw (i.e. non-metric) ``value`` is logged once per batch and the values are reduced to a
183
+ single number at the end of the epoch. Use the default ``reduce_fn="mean"`` when each
184
+ batch measures the same quantity, and ``"sum"`` when each batch contributes part of a
185
+ whole::
186
+
187
+ self.log('train_loss', loss) # typical loss over the epoch
188
+ self.log('samples_seen', len(batch), reduce_fn='sum') # samples seen this epoch
189
+
190
+ Nothing else is assumed about the value: it is taken at face value and, in particular, is
191
+ not rescaled by the batch size. Log a `reax.Metric` if the reduction needs to account for
192
+ what the numbers actually are, e.g. an average over samples rather than over batches.
193
+
194
+ Metric instances know how to combine themselves, so ``reduce_fn`` doesn't apply to them.
180
195
  """
181
196
  trainer = self._trainer
182
197
  if trainer is None:
@@ -206,6 +221,7 @@ class Module(
206
221
  logger=logger,
207
222
  on_step=on_step,
208
223
  on_epoch=on_epoch,
224
+ reduce_fn=reduce_fn,
209
225
  )
210
226
 
211
227
  def log_dict(
@@ -38,7 +38,7 @@ def get_batch_sizers() -> list:
38
38
  return sizers
39
39
 
40
40
 
41
- def get_metrics() -> "dict[str, reax.typing.MetricType]":
41
+ def get_metrics() -> "dict[str, reax.types.MetricType]":
42
42
  """Get registered metrics from plugins"""
43
43
  mgr = stevedore.extension.ExtensionManager(
44
44
  namespace="reax.plugins.metrics",
@@ -46,7 +46,7 @@ def get_metrics() -> "dict[str, reax.typing.MetricType]":
46
46
  on_load_failure_callback=load_failed,
47
47
  )
48
48
 
49
- metrics: "dict[str, reax.typing.MetricType]" = {}
49
+ metrics: "dict[str, reax.types.MetricType]" = {}
50
50
 
51
51
  def get_type(extension: stevedore.extension.Extension):
52
52
  try:
@@ -1,6 +1,7 @@
1
1
  import dataclasses
2
2
  from typing import TYPE_CHECKING, Generic, TypeVar
3
3
 
4
+ import equinox
4
5
  import jax
5
6
  import jax.numpy as jnp
6
7
  import jaxtyping as jt
@@ -28,51 +29,98 @@ class Metadata:
28
29
 
29
30
 
30
31
  class ArrayResultMetric(_metric.Metric[jax.Array]):
32
+ """Accumulate raw logged values, reducing them over the batches of an epoch.
33
+
34
+ With ``reduce_fn="sum"`` the values are simply added up. With ``"mean"`` each is weighted by
35
+ the ``batch_size`` it was logged with, so the result is the average over all the samples seen
36
+ rather than over the batches, and a short final batch counts for the samples it actually holds.
37
+ Where the batch size isn't known every value counts once, giving the mean over the batches.
38
+ """
39
+
31
40
  value: jax.Array = 0 # Redefine here so the typing hinting works
32
- cumulated_batch_size: int = 0
41
+ weight: jax.Array = 0
42
+ reduce_fn: "reax.types.ReduceFn" = equinox.field(static=True, default="mean")
33
43
 
34
44
  @classmethod
35
45
  def create(
36
46
  # pylint: disable=arguments-differ
37
47
  cls,
38
48
  value: jt.ArrayLike,
39
- batch_size: int,
49
+ batch_size: int | None = None,
50
+ reduce_fn: "reax.types.ReduceFn" = "mean",
40
51
  ) -> "ArrayResultMetric":
41
52
  """Create function."""
42
- return ArrayResultMetric(value=jnp.asarray(value), cumulated_batch_size=batch_size)
53
+ reduce_fn = _check_reduce_fn(reduce_fn)
54
+ weight = _weight(batch_size, reduce_fn)
55
+
56
+ return ArrayResultMetric(
57
+ value=jnp.asarray(value) * weight, weight=weight, reduce_fn=reduce_fn
58
+ )
43
59
 
44
60
  @override
45
61
  def update(
46
62
  # pylint: disable=arguments-differ
47
63
  self,
48
64
  value: jt.ArrayLike,
49
- batch_size: int,
65
+ batch_size: int | None = None,
50
66
  ) -> "ArrayResultMetric":
51
67
  """Update function."""
68
+ weight = _weight(batch_size, self.reduce_fn)
69
+
52
70
  return ArrayResultMetric(
53
- value=self.value + jnp.asarray(value),
54
- cumulated_batch_size=self.cumulated_batch_size + batch_size,
71
+ value=self.value + jnp.asarray(value) * weight,
72
+ weight=self.weight + weight,
73
+ reduce_fn=self.reduce_fn,
55
74
  )
56
75
 
57
76
  @override
58
77
  def merge(self, other: "ArrayResultMetric") -> "ArrayResultMetric":
59
78
  """Merge function."""
79
+ if other.reduce_fn != self.reduce_fn:
80
+ raise ValueError(
81
+ f"Cannot merge results that are reduced differently, got '{self.reduce_fn}' and "
82
+ f"'{other.reduce_fn}'"
83
+ )
84
+
60
85
  return ArrayResultMetric(
61
86
  value=self.value + other.value,
62
- cumulated_batch_size=self.cumulated_batch_size + other.cumulated_batch_size,
87
+ weight=self.weight + other.weight,
88
+ reduce_fn=self.reduce_fn,
63
89
  )
64
90
 
65
91
  @override
66
92
  def compute(self) -> jax.Array:
67
93
  """Compute function."""
68
- return self.value / self.cumulated_batch_size
94
+ if self.reduce_fn == "sum":
95
+ return self.value
96
+
97
+ return self.value / self.weight
98
+
99
+
100
+ def _weight(batch_size: int | None, reduce_fn: "reax.types.ReduceFn") -> int:
101
+ """How much a value logged with ``batch_size`` counts for.
102
+
103
+ Totals are added up as they are, so they carry no weight. Averages are weighted by the number
104
+ of samples behind them, falling back to counting once each when that isn't known.
105
+ """
106
+ if reduce_fn == "sum" or batch_size is None:
107
+ return 1
108
+
109
+ return batch_size
110
+
111
+
112
+ def _check_reduce_fn(reduce_fn: "reax.types.ReduceFn") -> "reax.types.ReduceFn":
113
+ if reduce_fn not in ("sum", "mean"):
114
+ raise ValueError(f"`reduce_fn` must be one of 'sum' or 'mean', got {reduce_fn!r}")
115
+
116
+ return reduce_fn
69
117
 
70
118
 
71
119
  class ResultEntry(Generic[_OutT]):
72
120
  def __init__(
73
121
  self,
74
122
  meta: Metadata,
75
- metric: "reax.typing.MetricInstance[_OutT]",
123
+ metric: "reax.types.MetricInstance[_OutT]",
76
124
  last_value: "reax.types.MetricInstance[_OutT] | None" = None,
77
125
  ):
78
126
  """Init function."""
@@ -107,7 +155,7 @@ class ResultCollection(dict[str, ResultEntry]):
107
155
  self,
108
156
  fx: str,
109
157
  name: str,
110
- value: "jt.ArrayLike | reax.typing.MetricInstance[_OutT]",
158
+ value: "jt.ArrayLike | reax.types.MetricInstance[_OutT]",
111
159
  batch_idx: int,
112
160
  *,
113
161
  prog_bar: bool = False,
@@ -115,15 +163,21 @@ class ResultCollection(dict[str, ResultEntry]):
115
163
  on_step: bool = False,
116
164
  on_epoch: bool = True,
117
165
  batch_size: int | None = None,
166
+ reduce_fn: "reax.types.ReduceFn" = "mean",
118
167
  ):
119
- """Log function."""
168
+ """Log function.
169
+
170
+ ``reduce_fn`` says how raw values logged over an epoch should be reduced to a single value,
171
+ with ``batch_size`` weighting them when taking their mean. Both are ignored for metric
172
+ instances, which know how to combine themselves.
173
+ """
120
174
  key = f"{fx}.{name}"
121
175
 
122
176
  if isinstance(value, types.MetricInstance):
123
177
  metric = value
124
178
  else:
125
179
  try:
126
- metric = ArrayResultMetric.create(value, batch_size)
180
+ metric = ArrayResultMetric.create(value, batch_size, reduce_fn)
127
181
  except TypeError:
128
182
  raise TypeError(
129
183
  f"Value must be a `reax.Metric` or a raw value, got {type(value).__name__}"
@@ -170,7 +170,7 @@ class DataSourceManager(Generic[_T_co]):
170
170
  ) -> "reax.data.DataLoader[_T_co, _U]":
171
171
  return self.dataloader.with_new_sampler(sampler)
172
172
 
173
- def __init__(self, source: "reax.data.DataSource[_T_co]"):
173
+ def __init__(self, source: "reax.data.DataSource[_T_co, _U]"):
174
174
  self._source = source
175
175
  self._ready = False
176
176
 
@@ -182,7 +182,7 @@ class DataSourceManager(Generic[_T_co]):
182
182
  return DataSourceManager.LoaderProxy(self, method_name)
183
183
 
184
184
  @property
185
- def source(self) -> "reax.data.DataSource[_T_co]":
185
+ def source(self) -> "reax.data.DataSource[_T_co, _U]":
186
186
  if not self._ready:
187
187
  raise RuntimeError(
188
188
  "`prepare_and_setup` has not been called, this must be done before accessing the "
@@ -201,7 +201,7 @@ class DataSourceManager(Generic[_T_co]):
201
201
 
202
202
 
203
203
  def get_datasource(
204
- datamodule: "reax.DataModule[_T_co] | None" = None,
204
+ datamodule: "reax.DataModule[_T_co, _U] | None" = None,
205
205
  module: "reax.Module | None" = None,
206
206
  ) -> DataSourceManager[_T_co] | None:
207
207
  if datamodule is not None:
@@ -186,7 +186,7 @@ class FitEpoch(train.Train):
186
186
 
187
187
  @property
188
188
  def val_dataloader(self) -> "reax.DataLoader | None":
189
- """Train dataloader."""
189
+ """Validation dataloader."""
190
190
  if self._validate is None:
191
191
  return None
192
192
 
@@ -252,18 +252,19 @@ class FitEpoch(train.Train):
252
252
  def log(
253
253
  self,
254
254
  name: str,
255
- value: "jt.ArrayLike | reax.typing.MetricInstance",
255
+ value: "jt.ArrayLike | reax.types.MetricInstance",
256
256
  batch_size: int | None = None,
257
257
  prog_bar: bool = False,
258
258
  logger: bool = False,
259
259
  on_step=False,
260
260
  on_epoch=True,
261
+ reduce_fn: "reax.types.ReduceFn" = "mean",
261
262
  ) -> None:
262
263
  """Log function."""
263
264
  if self._child is not None:
264
- self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
265
+ self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
265
266
  else:
266
- super().log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
267
+ super().log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
267
268
 
268
269
  def _should_check_val(self) -> bool:
269
270
  """Decide if we should run validation based on the current mode and progress."""
@@ -511,6 +512,7 @@ class Fit(stages.Stage):
511
512
  if not self.enable_validation:
512
513
  return None
513
514
 
515
+ assert self._fit_epoch.validate is not None
514
516
  return self._fit_epoch.validate.num_batches
515
517
 
516
518
  @property
@@ -530,11 +532,12 @@ class Fit(stages.Stage):
530
532
 
531
533
  @property
532
534
  def train_dataloaders(self) -> "reax.DataLoader":
535
+ """Training dataloader"""
533
536
  return self.train_dataloader
534
537
 
535
538
  @property
536
539
  def val_dataloader(self) -> "reax.DataLoader | None":
537
- """Train dataloader."""
540
+ """Validation dataloader."""
538
541
  return self._datamanager.get_dataloader("val")
539
542
 
540
543
  @property
@@ -555,6 +558,7 @@ class Fit(stages.Stage):
555
558
  logger: bool = False,
556
559
  on_step=False,
557
560
  on_epoch=True,
561
+ reduce_fn: "reax.types.ReduceFn" = "mean",
558
562
  ) -> None:
559
563
  """Log function."""
560
564
  if self._child is None:
@@ -563,7 +567,7 @@ class Fit(stages.Stage):
563
567
  "so cannot currently log anything"
564
568
  )
565
569
 
566
- self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
570
+ self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
567
571
 
568
572
  @override
569
573
  def _on_iteration_starting(self):
@@ -1,4 +1,4 @@
1
- from typing import TYPE_CHECKING, Any
1
+ from typing import TYPE_CHECKING, Any, Final
2
2
  import weakref
3
3
 
4
4
  import beartype
@@ -35,6 +35,8 @@ class Predict(stages.EpochStage):
35
35
  fast_dev_run=fast_dev_run,
36
36
  limit_batches=limit_batches,
37
37
  )
38
+ # Params
39
+ self._mod: "Final[reax.Module]" = module
38
40
  self._keep_predictions = keep_predictions
39
41
  self._all_outputs: list[Any] | list[list[Any]] | None = []
40
42
 
@@ -45,22 +47,22 @@ class Predict(stages.EpochStage):
45
47
  @override
46
48
  def _on_starting(self):
47
49
  super()._on_starting()
48
- self._module.on_predict_start(weakref.proxy(self))
50
+ self._mod.on_predict_start(weakref.proxy(self))
49
51
 
50
52
  @override
51
53
  def _on_epoch_start(self):
52
54
  super()._on_epoch_start()
53
- self._module.on_predict_epoch_start(weakref.proxy(self))
55
+ self._mod.on_predict_epoch_start(weakref.proxy(self))
54
56
 
55
57
  @override
56
58
  def _on_iteration_starting(self):
57
59
  super()._on_iteration_starting()
58
- self._module.on_predict_batch_start(self, self.batch, self.batch_idx)
60
+ self._mod.on_predict_batch_start(self, self.batch, self.batch_idx)
59
61
 
60
62
  @override
61
63
  def _step(self) -> "reax.stages.MetricResults":
62
64
  """Step function."""
63
- return self._module.predict_step(self.batch, self._iter)
65
+ return self._mod.predict_step(self.batch, self._iter)
64
66
 
65
67
  @override
66
68
  def _on_iteration_finishing(self, outputs: Any, /):
@@ -69,14 +71,14 @@ class Predict(stages.EpochStage):
69
71
  if self._keep_predictions:
70
72
  cpu = jax.devices("cpu")[0]
71
73
  self._all_outputs.append(jax.device_put(outputs, cpu))
72
- self._module.on_predict_batch_end(self, outputs, self.batch, self.batch_idx)
74
+ self._mod.on_predict_batch_end(self, outputs, self.batch, self.batch_idx)
73
75
 
74
76
  @override
75
77
  def _on_epoch_end(self) -> None:
76
78
  super()._on_epoch_end()
77
- self._module.on_predict_epoch_end(weakref.proxy(self))
79
+ self._mod.on_predict_epoch_end(weakref.proxy(self))
78
80
 
79
81
  @override
80
82
  def _on_stopped(self) -> None:
81
83
  super()._on_stopped()
82
- self._module.on_predict_end(weakref.proxy(self))
84
+ self._mod.on_predict_end(weakref.proxy(self))
@@ -252,6 +252,7 @@ class Stage(abc.ABC):
252
252
  logger: bool = False,
253
253
  on_step=False,
254
254
  on_epoch=True,
255
+ reduce_fn: "reax.types.ReduceFn" = "mean",
255
256
  ) -> None:
256
257
  """Log a result while the stage is running."""
257
258
 
@@ -391,12 +392,12 @@ class EpochStage(Stage, abc.ABC):
391
392
  self._outputs = None
392
393
 
393
394
  @property
394
- def dataloader(self) -> "reax.DataLoader | None":
395
+ def dataloader(self) -> "reax.DataLoader":
395
396
  """Dataloader function."""
396
397
  return self._datamanager.get_dataloader(self._dataloader_name)
397
398
 
398
399
  @property
399
- def dataloaders(self) -> "reax.DataLoader | None":
400
+ def dataloaders(self) -> "reax.DataLoader":
400
401
  """Dataloader function."""
401
402
  return self.dataloader
402
403
 
@@ -503,6 +504,7 @@ class EpochStage(Stage, abc.ABC):
503
504
  logger: bool = False,
504
505
  on_step=False,
505
506
  on_epoch=True,
507
+ reduce_fn: "reax.types.ReduceFn" = "mean",
506
508
  ) -> None:
507
509
  """Log metrics during the current epoch."""
508
510
  assert self._batch is not None
@@ -519,6 +521,7 @@ class EpochStage(Stage, abc.ABC):
519
521
  batch_idx=self._iter,
520
522
  on_step=on_step,
521
523
  on_epoch=on_epoch,
524
+ reduce_fn=reduce_fn,
522
525
  )
523
526
 
524
527
  @override
@@ -41,27 +41,29 @@ class Test(stages.EpochStage):
41
41
  fast_dev_run=fast_dev_run,
42
42
  limit_batches=limit_batches,
43
43
  )
44
+ # Params
45
+ self._mod: "reax.Module" = module
44
46
 
45
47
  @override
46
48
  def _on_starting(self):
47
49
  super()._on_starting()
48
- self._module.on_test_start(weakref.proxy(self))
50
+ self._mod.on_test_start(weakref.proxy(self))
49
51
 
50
52
  @override
51
53
  def _on_epoch_start(self):
52
54
  super()._on_epoch_start()
53
- self._module.on_test_epoch_start(weakref.proxy(self))
55
+ self._mod.on_test_epoch_start(weakref.proxy(self))
54
56
 
55
57
  @override
56
58
  def _step(self) -> "reax.stages.MetricResults":
57
- return self._module.test_step(self.batch, self._iter)
59
+ return self._mod.test_step(self.batch, self._iter)
58
60
 
59
61
  @override
60
62
  def _on_epoch_end(self):
61
63
  super()._on_epoch_end()
62
- self._module.on_test_epoch_end(weakref.proxy(self))
64
+ self._mod.on_test_epoch_end(weakref.proxy(self))
63
65
 
64
66
  @override
65
67
  def _on_stopping(self) -> None:
66
68
  super()._on_stopping()
67
- self._module.on_test_end(weakref.proxy(self))
69
+ self._mod.on_test_end(weakref.proxy(self))
@@ -54,6 +54,7 @@ class Train(stages.EpochStage):
54
54
  enable_checkpointing=True,
55
55
  )
56
56
  # Params
57
+ self._mod: "Final[reax.Module]" = module
57
58
  self._min_updates: Final[int] = min_updates
58
59
  self._max_updates: Final[int | float | None] = max_updates
59
60
  self._accumulate_grad_batches: Final[int] = accumulate_grad_batches
@@ -73,12 +74,12 @@ class Train(stages.EpochStage):
73
74
  return sum(opt.update_count for opt in self._optimizers)
74
75
 
75
76
  @property
76
- def optimizers(self) -> list["reax.Optimizer"] | None:
77
+ def optimizers(self) -> "list[reax.Optimizer] | None":
77
78
  """Optimizers function."""
78
79
  return self._optimizers
79
80
 
80
81
  @override
81
- def run(self) -> list["reax.Optimizer"]:
82
+ def run(self) -> "list[reax.Optimizer]":
82
83
  """Run function."""
83
84
  super().run()
84
85
  return self._optimizers
@@ -89,14 +90,14 @@ class Train(stages.EpochStage):
89
90
  super()._on_starting()
90
91
 
91
92
  if not self._optimizers:
92
- opts = self._module.configure_optimizers()
93
+ opts = self._mod.configure_optimizers()
93
94
  if opts is None:
94
95
  rank_zero.rank_zero_warn(
95
96
  "`reax.Module.configure_optimizers` returned `None`, this fit will run "
96
97
  "with no optimizer"
97
98
  )
98
99
  opt = optimizers_.mock_optimizer
99
- opts = opt, opt.init(self._module.parameters())
100
+ opts = opt, opt.init(self._mod.parameters())
100
101
 
101
102
  if not isinstance(opts, list):
102
103
  opts = [opts]
@@ -107,7 +108,7 @@ class Train(stages.EpochStage):
107
108
  state = self._engine.to_device(state)
108
109
  if self._accumulate_grad_batches > 1:
109
110
  stepper = optax.MultiSteps(opt, every_k_schedule=self._accumulate_grad_batches)
110
- state = stepper.init(self._module.parameters())
111
+ state = stepper.init(self._mod.parameters())
111
112
  opt = stepper.gradient_transformation()
112
113
 
113
114
  optimizers.append(optimizers_.Optimizer(opt, state))
@@ -115,38 +116,38 @@ class Train(stages.EpochStage):
115
116
  # Create the `Optimizer` instances
116
117
  self._optimizers = optimizers
117
118
 
118
- self._module.on_train_start(weakref.proxy(self))
119
+ self._mod.on_train_start(weakref.proxy(self))
119
120
 
120
121
  @override
121
122
  def _on_iteration_starting(self):
122
123
  super()._on_iteration_starting()
123
- self._module.on_train_batch_start(weakref.proxy(self), self.batch, self.batch_idx)
124
+ self._mod.on_train_batch_start(weakref.proxy(self), self.batch, self.batch_idx)
124
125
 
125
126
  @override
126
127
  def _on_epoch_start(self):
127
128
  """On started."""
128
129
  super()._on_epoch_start()
129
- self._module.on_train_epoch_start(weakref.proxy(self))
130
+ self._mod.on_train_epoch_start(weakref.proxy(self))
130
131
 
131
132
  @override
132
133
  def _step(self) -> Any:
133
134
  """Step function."""
134
- if self._module.parameters() is None:
135
+ if self._mod.parameters() is None:
135
136
  raise exceptions.MisconfigurationException(
136
137
  "Module does not have any parameters set, this should have been done in "
137
138
  ".configure_model()."
138
139
  )
139
140
 
140
- res = self._module.training_step(self.batch, self._iter)
141
- if self._module.automatic_optimization:
141
+ res = self._mod.training_step(self.batch, self._iter)
142
+ if self._mod.automatic_optimization:
142
143
  if isinstance(res, dict):
143
144
  grad = res["grad"]
144
145
  loss = res.get("loss", None)
145
146
  else:
146
147
  loss, grad = res
147
148
  opt = self._optimizers[0]
148
- self._module.on_before_optimizer_step(opt, grad)
149
- opt = opt.update_module(self._module, grad, value=loss)
149
+ self._mod.on_before_optimizer_step(opt, grad)
150
+ opt = opt.update_module(self._mod, grad, value=loss)
150
151
  self._optimizers = [opt]
151
152
 
152
153
  if (self._min_updates is None or self.updates >= self._min_updates) and (
@@ -159,17 +160,17 @@ class Train(stages.EpochStage):
159
160
  @override
160
161
  def _on_iteration_finishing(self, outputs: Any, /):
161
162
  super()._on_iteration_finishing(outputs)
162
- self._module.on_train_batch_end(weakref.proxy(self), outputs, self.batch, self.batch_idx)
163
+ self._mod.on_train_batch_end(weakref.proxy(self), outputs, self.batch, self.batch_idx)
163
164
 
164
165
  @override
165
166
  def _on_epoch_end(self) -> None:
166
167
  super()._on_epoch_end()
167
- self._module.on_train_epoch_end(weakref.proxy(self))
168
+ self._mod.on_train_epoch_end(weakref.proxy(self))
168
169
 
169
170
  @override
170
171
  def _on_stopping(self) -> None:
171
172
  super()._on_stopping()
172
- self._module.on_train_end(weakref.proxy(self))
173
+ self._mod.on_train_end(weakref.proxy(self))
173
174
 
174
175
  @override
175
176
  def _done(self) -> bool:
@@ -192,12 +193,14 @@ class Train(stages.EpochStage):
192
193
  f"`{type(self).__name__}` stopped: `{type(self).__name__}.should_stop` was set."
193
194
  )
194
195
  else:
195
- self._warning_cache.info(
196
- f"Trainer was signaled to stop but the required "
197
- f"`min_epochs={self.parent.min_iters!r}` or"
196
+ lines = ["Trainer was signaled to stop but the required "]
197
+ if self.parent is not None:
198
+ lines.append(f"`min_epochs={self.parent.min_iters!r}` or")
199
+ lines.append(
198
200
  f" `min_steps={self._min_updates!r}` has not been met. "
199
201
  f"Training will continue..."
200
202
  )
203
+ self._warning_cache.info("".join(lines))
201
204
  return self._stopper.can_stop
202
205
 
203
206
  return False
@@ -46,6 +46,9 @@ class Validate(stages.EpochStage):
46
46
  enable_checkpointing=enable_checkpointing,
47
47
  )
48
48
 
49
+ # Params
50
+ self._mod: "reax.Module" = module
51
+
49
52
  @property
50
53
  def epoch(self) -> int:
51
54
  """Get the current epoch."""
@@ -54,34 +57,34 @@ class Validate(stages.EpochStage):
54
57
  @override
55
58
  def _on_starting(self):
56
59
  super()._on_starting()
57
- self._module.on_validation_start(weakref.proxy(self))
60
+ self._mod.on_validation_start(weakref.proxy(self))
58
61
 
59
62
  @override
60
63
  def _on_epoch_start(self):
61
64
  super()._on_epoch_start()
62
- self._module.on_validation_epoch_start(weakref.proxy(self))
65
+ self._mod.on_validation_epoch_start(weakref.proxy(self))
63
66
 
64
67
  @override
65
68
  def _on_iteration_starting(self):
66
69
  super()._on_iteration_starting()
67
- self._module.on_validation_batch_start(self, self.batch, self.batch_idx)
70
+ self._mod.on_validation_batch_start(self, self.batch, self.batch_idx)
68
71
 
69
72
  @override
70
73
  def _step(self) -> "reax.stages.MetricResults":
71
- return self._module.validation_step(self.batch, self._iter)
74
+ return self._mod.validation_step(self.batch, self._iter)
72
75
 
73
76
  @override
74
77
  def _on_iteration_finishing(self, outputs: Any, /):
75
78
  """On iteration finishing."""
76
79
  super()._on_iteration_finishing(outputs)
77
- self._module.on_validation_batch_end(self, outputs, self.batch, self.batch_idx)
80
+ self._mod.on_validation_batch_end(self, outputs, self.batch, self.batch_idx)
78
81
 
79
82
  @override
80
83
  def _on_epoch_end(self) -> None:
81
84
  super()._on_epoch_end()
82
- self._module.on_validation_epoch_end(weakref.proxy(self))
85
+ self._mod.on_validation_epoch_end(weakref.proxy(self))
83
86
 
84
87
  @override
85
88
  def _on_stopping(self) -> None:
86
89
  super()._on_stopping()
87
- self._module.on_validation_end(weakref.proxy(self))
90
+ self._mod.on_validation_end(weakref.proxy(self))
@@ -230,13 +230,13 @@ class JaxDdpStrategy(_parallel.ParallelStrategy):
230
230
  multihost_utils.sync_global_devices(name)
231
231
 
232
232
  @override
233
- def compute(self, metric: "reax.typing.MetricInstance[_OutT]") -> _OutT:
233
+ def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
234
234
  if self.process_count == 1:
235
235
  return metric.compute()
236
236
 
237
237
  dynamic, static = eqx.partition(metric, eqx.is_array)
238
238
  gathered = multihost_utils.process_allgather(dynamic)
239
- unbatched: "list[reax.typing.MetricInstance[_OutT]]" = unbatch_pytree(gathered, dynamic)
239
+ unbatched: "list[reax.types.MetricInstance[_OutT]]" = unbatch_pytree(gathered, dynamic)
240
240
  # Rejoin with the static data
241
241
  unbatched = [eqx.combine(entry, static) for entry in unbatched]
242
242
 
@@ -76,5 +76,5 @@ class SingleDevice(_strategies.Strategy):
76
76
  return None # Nothing to do in a single device
77
77
 
78
78
  @override
79
- def compute(self, metric: "reax.typing.MetricInstance[_OutT]") -> _OutT:
79
+ def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
80
80
  return metric.compute()
@@ -123,7 +123,7 @@ class Strategy(abc.ABC):
123
123
  return data_.ReaxDataLoader(data)
124
124
 
125
125
  @abc.abstractmethod
126
- def compute(self, metric: "reax.typing.MetricInstance[_OutT]") -> _OutT:
126
+ def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
127
127
  """Compute the value of a metric, unlike metric.compute(), in a parallel setting this method
128
128
  will compute the value across all processes.
129
129
  """
@@ -395,6 +395,7 @@ class Trainer(stages.StageListener, _deprecated.TrainerDeprecatedMixin):
395
395
  logger: bool = None,
396
396
  on_step=True,
397
397
  on_epoch=True,
398
+ reduce_fn: "reax.types.ReduceFn" = "mean",
398
399
  ) -> None:
399
400
  """Log function."""
400
401
  if self._stage is None:
@@ -412,6 +413,7 @@ class Trainer(stages.StageListener, _deprecated.TrainerDeprecatedMixin):
412
413
  batch_size=batch_size,
413
414
  on_step=on_step,
414
415
  on_epoch=on_epoch,
416
+ reduce_fn=reduce_fn,
415
417
  )
416
418
 
417
419
  # region Stages
@@ -1,5 +1,5 @@
1
1
  import pathlib
2
- from typing import Protocol, TypeVar, runtime_checkable
2
+ from typing import Literal, Protocol, TypeVar, runtime_checkable
3
3
 
4
4
  import jax.typing
5
5
  import jaxtyping as jt
@@ -11,6 +11,10 @@ Path = str | bytes | pathlib.Path
11
11
  ArrayMask = jt.Int[jt.ArrayLike, "..."] | jt.Bool[jt.ArrayLike, "..."]
12
12
  MetricsDict = dict[str, jax.typing.ArrayLike]
13
13
 
14
+ #: How the raw (i.e. non-metric) values logged over the batches of an epoch should be reduced to a
15
+ #: single value at the end of it: either their mean, or their total.
16
+ ReduceFn = Literal["sum", "mean"]
17
+
14
18
 
15
19
  @runtime_checkable
16
20
  class MetricType(Protocol[_OutT]):
File without changes
File without changes
File without changes
File without changes
File without changes