reax 0.6.13.dev1__tar.gz → 0.6.14.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/PKG-INFO +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/__init__.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/_engine.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_datasources.py +6 -5
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_loaders.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_types.py +7 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/datamodules.py +11 -8
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/utils.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/modules.py +19 -3
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/plugins.py +2 -2
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/results.py +66 -12
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/common.py +3 -3
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/fit.py +10 -6
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/predict.py +10 -8
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/stages.py +5 -2
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/test.py +7 -5
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/train.py +22 -19
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/validation.py +10 -7
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_jax.py +2 -2
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_single_device.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_strategies.py +1 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/trainer.py +2 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/types.py +5 -1
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/LICENSE.txt +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/README.rst +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/pyproject.toml +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/_module_hooks.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/_datasource_manager.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/collate.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/dataloaders.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/datasets.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/fetchers.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/kfold.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/data/samplers.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/boring_classes.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/demos/mnist.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/exceptions.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/hooks.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/keys.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/LICENSE +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/lightning/rank_zero.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/checkpointer.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/early_stopping.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/model_checkpoint.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/progress_bar.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/rich_progress.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/progress/tqdm_progress.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/listeners/utils.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/_utils.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/csv_logs.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/logger.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/mlflow.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/pandas.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/loggers/tensorboard.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_classification.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_evaluators.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_globals.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_metric.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/_registry.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/aggregation.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/collections.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/types.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/jm/utils.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/regression.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/metrics/utils.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/optimizers.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/jax_profiler.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/profilers/profiler.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/random.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/saving.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/_timer.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/stages/stats.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_factory.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_parallel.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/strategies/_utils.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/testing.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_checkpoint_connector.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_checkpointing.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_deprecated.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/training/_logger_connector.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/__init__.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/arrays.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/containers.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/events.py +0 -0
- {reax-0.6.13.dev1 → reax-0.6.14.dev1}/src/reax/utils/grads.py +0 -0
|
@@ -415,7 +415,7 @@ class Engine:
|
|
|
415
415
|
"""
|
|
416
416
|
self._events.fire_event(getattr(hooks.TrainerListener, name), *args, **kwargs)
|
|
417
417
|
|
|
418
|
-
def compute(self, metric: "reax.
|
|
418
|
+
def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
|
|
419
419
|
"""
|
|
420
420
|
Computes the final value of a metric across all distributed processes.
|
|
421
421
|
|
|
@@ -8,9 +8,10 @@ if TYPE_CHECKING:
|
|
|
8
8
|
__all__ = ("DataSource",)
|
|
9
9
|
|
|
10
10
|
_T_co = TypeVar("_T_co", covariant=True)
|
|
11
|
+
U = TypeVar("U")
|
|
11
12
|
|
|
12
13
|
|
|
13
|
-
class DataSource(Generic[_T_co]):
|
|
14
|
+
class DataSource(Generic[_T_co, U]):
|
|
14
15
|
"""Interface for classes that provide datasets."""
|
|
15
16
|
|
|
16
17
|
def __init__(self) -> None:
|
|
@@ -117,7 +118,7 @@ class DataSource(Generic[_T_co]):
|
|
|
117
118
|
|
|
118
119
|
# endregion
|
|
119
120
|
|
|
120
|
-
def train_dataloader(self) -> "reax.DataLoader[_T_co]":
|
|
121
|
+
def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
121
122
|
"""An iterable or collection of iterables specifying training samples.
|
|
122
123
|
|
|
123
124
|
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
@@ -148,7 +149,7 @@ class DataSource(Generic[_T_co]):
|
|
|
148
149
|
"`train_dataloader` must be implemented to be used with the REAX Trainer"
|
|
149
150
|
)
|
|
150
151
|
|
|
151
|
-
def val_dataloader(self) -> Optional["reax.DataLoader[_T_co]"]:
|
|
152
|
+
def val_dataloader(self) -> Optional["reax.DataLoader[_T_co, U]"]:
|
|
152
153
|
r"""An iterable or collection of iterables specifying validation samples.
|
|
153
154
|
|
|
154
155
|
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
@@ -174,7 +175,7 @@ class DataSource(Generic[_T_co]):
|
|
|
174
175
|
|
|
175
176
|
"""
|
|
176
177
|
|
|
177
|
-
def test_dataloader(self) -> "reax.DataLoader[_T_co]":
|
|
178
|
+
def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
178
179
|
r"""An iterable or collection of iterables specifying test samples.
|
|
179
180
|
|
|
180
181
|
For more information about multiple dataloaders, this :ref:`section <multiple-dataloaders>`.
|
|
@@ -206,7 +207,7 @@ class DataSource(Generic[_T_co]):
|
|
|
206
207
|
"`test_dataloader` must be implemented to be used with the REAX Trainer"
|
|
207
208
|
)
|
|
208
209
|
|
|
209
|
-
def predict_dataloader(self) -> "reax.DataLoader[_T_co]":
|
|
210
|
+
def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
210
211
|
r"""An iterable or collection of iterables specifying prediction samples.
|
|
211
212
|
|
|
212
213
|
For more information about multiple dataloaders, see :ref:`section <multiple-dataloaders>`.
|
|
@@ -238,5 +238,5 @@ class DeviceDataLoader(_types.DataLoader[_T_co, _U]):
|
|
|
238
238
|
return self._loader.sampler
|
|
239
239
|
|
|
240
240
|
@override
|
|
241
|
-
def with_new_sampler(self, sampler: "reax.data.Sampler") -> "
|
|
241
|
+
def with_new_sampler(self, sampler: "reax.data.Sampler") -> "DeviceDataLoader[_T_co, _U]":
|
|
242
242
|
return DeviceDataLoader(self._loader.with_new_sampler(sampler), device=self._device)
|
|
@@ -17,6 +17,13 @@ CollateFn = Callable[[Sequence[_T_co]], U]
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class DataLoader(Generic[_T_co, U], Iterable[U], abc.ABC):
|
|
20
|
+
"""Abstract base class for loaders that yield processed batches of data.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
_T_co: The type of elements in the underlying dataset.
|
|
24
|
+
U: The type of the batches produced by the loader.
|
|
25
|
+
"""
|
|
26
|
+
|
|
20
27
|
def __len__(self) -> int:
|
|
21
28
|
return len(self.sampler)
|
|
22
29
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from collections.abc import Iterable
|
|
2
|
-
from typing import TYPE_CHECKING, Any
|
|
2
|
+
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
|
3
3
|
|
|
4
4
|
from flax import nnx
|
|
5
5
|
from typing_extensions import override
|
|
@@ -13,8 +13,11 @@ __all__ = ("DataModule",)
|
|
|
13
13
|
|
|
14
14
|
Dataset = Any
|
|
15
15
|
|
|
16
|
+
_T_co = TypeVar("_T_co", covariant=True)
|
|
17
|
+
U = TypeVar("U")
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
|
|
20
|
+
class DataModule(Generic[_T_co, U], _datasources.DataSource[_T_co, U]):
|
|
18
21
|
def __init__(self):
|
|
19
22
|
super().__init__()
|
|
20
23
|
self._rngs = nnx.Rngs(0)
|
|
@@ -36,14 +39,14 @@ class DataModule(_datasources.DataSource):
|
|
|
36
39
|
predict_dataset: Dataset | Iterable[Dataset] | None = None,
|
|
37
40
|
*,
|
|
38
41
|
batch_size: int = 1,
|
|
39
|
-
) -> "DataModule":
|
|
42
|
+
) -> "DataModule[_T_co, U]":
|
|
40
43
|
"""From datasets."""
|
|
41
44
|
return FromDatasets(
|
|
42
45
|
train_dataset, val_dataset, test_dataset, predict_dataset, batch_size=batch_size
|
|
43
46
|
)
|
|
44
47
|
|
|
45
48
|
|
|
46
|
-
class FromDatasets(DataModule):
|
|
49
|
+
class FromDatasets(DataModule[_T_co, U], Generic[_T_co, U]):
|
|
47
50
|
def __init__(
|
|
48
51
|
self,
|
|
49
52
|
train_dataset: Dataset | Iterable[Dataset] | None = None,
|
|
@@ -62,21 +65,21 @@ class FromDatasets(DataModule):
|
|
|
62
65
|
self._batch_size = batch_size
|
|
63
66
|
|
|
64
67
|
@override
|
|
65
|
-
def train_dataloader(self) -> "reax.DataLoader":
|
|
68
|
+
def train_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
66
69
|
"""Train dataloader."""
|
|
67
70
|
return _loaders.ReaxDataLoader(self._train_dataset, batch_size=self._batch_size)
|
|
68
71
|
|
|
69
72
|
@override
|
|
70
|
-
def val_dataloader(self) -> "reax.DataLoader":
|
|
73
|
+
def val_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
71
74
|
"""Val dataloader."""
|
|
72
75
|
return _loaders.ReaxDataLoader(self._val_dataset, batch_size=self._batch_size)
|
|
73
76
|
|
|
74
77
|
@override
|
|
75
|
-
def test_dataloader(self) -> "reax.DataLoader":
|
|
78
|
+
def test_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
76
79
|
"""Test dataloader."""
|
|
77
80
|
return _loaders.ReaxDataLoader(self._test_dataset, batch_size=self._batch_size)
|
|
78
81
|
|
|
79
82
|
@override
|
|
80
|
-
def predict_dataloader(self) -> "reax.DataLoader":
|
|
83
|
+
def predict_dataloader(self) -> "reax.DataLoader[_T_co, U]":
|
|
81
84
|
"""Predict dataloader."""
|
|
82
85
|
return _loaders.ReaxDataLoader(self._predict_dataset, batch_size=self._batch_size)
|
|
@@ -22,7 +22,7 @@ class BatchSizer:
|
|
|
22
22
|
"""Tool for extracting batch sizes from dataset."""
|
|
23
23
|
|
|
24
24
|
def __init__(self):
|
|
25
|
-
self._registry
|
|
25
|
+
self._registry: containers.TypeRegistry[Extractor] = containers.TypeRegistry()
|
|
26
26
|
|
|
27
27
|
def register(
|
|
28
28
|
self,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from collections.abc import Callable, Generator, Sequence
|
|
2
2
|
import contextlib
|
|
3
|
-
from typing import TYPE_CHECKING, Any, Generic, TypedDict, TypeVar
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Generic, TypeAlias, TypedDict, TypeVar
|
|
4
4
|
|
|
5
5
|
import beartype
|
|
6
6
|
from flax import nnx
|
|
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
|
|
|
18
18
|
|
|
19
19
|
__all__ = ("Module",)
|
|
20
20
|
|
|
21
|
-
MetricType = "reax.Metric
|
|
21
|
+
MetricType: TypeAlias = "reax.Metric | jax.typing.ArrayLike"
|
|
22
22
|
OutputT_co = TypeVar("OutputT_co", covariant=True)
|
|
23
23
|
BatchT = TypeVar("BatchT")
|
|
24
24
|
OptimizerData = tuple[optax.GradientTransformation, Any]
|
|
@@ -34,7 +34,7 @@ TrainOutput = LossAndGrad | LossAndGradDict
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
class Module(
|
|
37
|
-
Generic[BatchT, OutputT_co], _module_hooks.ModuleHooks, _datasources.DataSource[BatchT]
|
|
37
|
+
Generic[BatchT, OutputT_co], _module_hooks.ModuleHooks, _datasources.DataSource[BatchT, Any]
|
|
38
38
|
):
|
|
39
39
|
example_input_array: BatchT | None
|
|
40
40
|
|
|
@@ -171,12 +171,27 @@ class Module(
|
|
|
171
171
|
logger: bool | None = None,
|
|
172
172
|
on_step=True,
|
|
173
173
|
on_epoch=True,
|
|
174
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
174
175
|
) -> None:
|
|
175
176
|
"""Log a key, value pair.
|
|
176
177
|
|
|
177
178
|
Example::
|
|
178
179
|
|
|
179
180
|
self.log('train_loss', loss)
|
|
181
|
+
|
|
182
|
+
A raw (i.e. non-metric) ``value`` is logged once per batch and the values are reduced to a
|
|
183
|
+
single number at the end of the epoch. Use the default ``reduce_fn="mean"`` when each
|
|
184
|
+
batch measures the same quantity, and ``"sum"`` when each batch contributes part of a
|
|
185
|
+
whole::
|
|
186
|
+
|
|
187
|
+
self.log('train_loss', loss) # typical loss over the epoch
|
|
188
|
+
self.log('samples_seen', len(batch), reduce_fn='sum') # samples seen this epoch
|
|
189
|
+
|
|
190
|
+
Nothing else is assumed about the value: it is taken at face value and, in particular, is
|
|
191
|
+
not rescaled by the batch size. Log a `reax.Metric` if the reduction needs to account for
|
|
192
|
+
what the numbers actually are, e.g. an average over samples rather than over batches.
|
|
193
|
+
|
|
194
|
+
Metric instances know how to combine themselves, so ``reduce_fn`` doesn't apply to them.
|
|
180
195
|
"""
|
|
181
196
|
trainer = self._trainer
|
|
182
197
|
if trainer is None:
|
|
@@ -206,6 +221,7 @@ class Module(
|
|
|
206
221
|
logger=logger,
|
|
207
222
|
on_step=on_step,
|
|
208
223
|
on_epoch=on_epoch,
|
|
224
|
+
reduce_fn=reduce_fn,
|
|
209
225
|
)
|
|
210
226
|
|
|
211
227
|
def log_dict(
|
|
@@ -38,7 +38,7 @@ def get_batch_sizers() -> list:
|
|
|
38
38
|
return sizers
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def get_metrics() -> "dict[str, reax.
|
|
41
|
+
def get_metrics() -> "dict[str, reax.types.MetricType]":
|
|
42
42
|
"""Get registered metrics from plugins"""
|
|
43
43
|
mgr = stevedore.extension.ExtensionManager(
|
|
44
44
|
namespace="reax.plugins.metrics",
|
|
@@ -46,7 +46,7 @@ def get_metrics() -> "dict[str, reax.typing.MetricType]":
|
|
|
46
46
|
on_load_failure_callback=load_failed,
|
|
47
47
|
)
|
|
48
48
|
|
|
49
|
-
metrics: "dict[str, reax.
|
|
49
|
+
metrics: "dict[str, reax.types.MetricType]" = {}
|
|
50
50
|
|
|
51
51
|
def get_type(extension: stevedore.extension.Extension):
|
|
52
52
|
try:
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import dataclasses
|
|
2
2
|
from typing import TYPE_CHECKING, Generic, TypeVar
|
|
3
3
|
|
|
4
|
+
import equinox
|
|
4
5
|
import jax
|
|
5
6
|
import jax.numpy as jnp
|
|
6
7
|
import jaxtyping as jt
|
|
@@ -28,51 +29,98 @@ class Metadata:
|
|
|
28
29
|
|
|
29
30
|
|
|
30
31
|
class ArrayResultMetric(_metric.Metric[jax.Array]):
|
|
32
|
+
"""Accumulate raw logged values, reducing them over the batches of an epoch.
|
|
33
|
+
|
|
34
|
+
With ``reduce_fn="sum"`` the values are simply added up. With ``"mean"`` each is weighted by
|
|
35
|
+
the ``batch_size`` it was logged with, so the result is the average over all the samples seen
|
|
36
|
+
rather than over the batches, and a short final batch counts for the samples it actually holds.
|
|
37
|
+
Where the batch size isn't known every value counts once, giving the mean over the batches.
|
|
38
|
+
"""
|
|
39
|
+
|
|
31
40
|
value: jax.Array = 0 # Redefine here so the typing hinting works
|
|
32
|
-
|
|
41
|
+
weight: jax.Array = 0
|
|
42
|
+
reduce_fn: "reax.types.ReduceFn" = equinox.field(static=True, default="mean")
|
|
33
43
|
|
|
34
44
|
@classmethod
|
|
35
45
|
def create(
|
|
36
46
|
# pylint: disable=arguments-differ
|
|
37
47
|
cls,
|
|
38
48
|
value: jt.ArrayLike,
|
|
39
|
-
batch_size: int,
|
|
49
|
+
batch_size: int | None = None,
|
|
50
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
40
51
|
) -> "ArrayResultMetric":
|
|
41
52
|
"""Create function."""
|
|
42
|
-
|
|
53
|
+
reduce_fn = _check_reduce_fn(reduce_fn)
|
|
54
|
+
weight = _weight(batch_size, reduce_fn)
|
|
55
|
+
|
|
56
|
+
return ArrayResultMetric(
|
|
57
|
+
value=jnp.asarray(value) * weight, weight=weight, reduce_fn=reduce_fn
|
|
58
|
+
)
|
|
43
59
|
|
|
44
60
|
@override
|
|
45
61
|
def update(
|
|
46
62
|
# pylint: disable=arguments-differ
|
|
47
63
|
self,
|
|
48
64
|
value: jt.ArrayLike,
|
|
49
|
-
batch_size: int,
|
|
65
|
+
batch_size: int | None = None,
|
|
50
66
|
) -> "ArrayResultMetric":
|
|
51
67
|
"""Update function."""
|
|
68
|
+
weight = _weight(batch_size, self.reduce_fn)
|
|
69
|
+
|
|
52
70
|
return ArrayResultMetric(
|
|
53
|
-
value=self.value + jnp.asarray(value),
|
|
54
|
-
|
|
71
|
+
value=self.value + jnp.asarray(value) * weight,
|
|
72
|
+
weight=self.weight + weight,
|
|
73
|
+
reduce_fn=self.reduce_fn,
|
|
55
74
|
)
|
|
56
75
|
|
|
57
76
|
@override
|
|
58
77
|
def merge(self, other: "ArrayResultMetric") -> "ArrayResultMetric":
|
|
59
78
|
"""Merge function."""
|
|
79
|
+
if other.reduce_fn != self.reduce_fn:
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"Cannot merge results that are reduced differently, got '{self.reduce_fn}' and "
|
|
82
|
+
f"'{other.reduce_fn}'"
|
|
83
|
+
)
|
|
84
|
+
|
|
60
85
|
return ArrayResultMetric(
|
|
61
86
|
value=self.value + other.value,
|
|
62
|
-
|
|
87
|
+
weight=self.weight + other.weight,
|
|
88
|
+
reduce_fn=self.reduce_fn,
|
|
63
89
|
)
|
|
64
90
|
|
|
65
91
|
@override
|
|
66
92
|
def compute(self) -> jax.Array:
|
|
67
93
|
"""Compute function."""
|
|
68
|
-
|
|
94
|
+
if self.reduce_fn == "sum":
|
|
95
|
+
return self.value
|
|
96
|
+
|
|
97
|
+
return self.value / self.weight
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _weight(batch_size: int | None, reduce_fn: "reax.types.ReduceFn") -> int:
|
|
101
|
+
"""How much a value logged with ``batch_size`` counts for.
|
|
102
|
+
|
|
103
|
+
Totals are added up as they are, so they carry no weight. Averages are weighted by the number
|
|
104
|
+
of samples behind them, falling back to counting once each when that isn't known.
|
|
105
|
+
"""
|
|
106
|
+
if reduce_fn == "sum" or batch_size is None:
|
|
107
|
+
return 1
|
|
108
|
+
|
|
109
|
+
return batch_size
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _check_reduce_fn(reduce_fn: "reax.types.ReduceFn") -> "reax.types.ReduceFn":
|
|
113
|
+
if reduce_fn not in ("sum", "mean"):
|
|
114
|
+
raise ValueError(f"`reduce_fn` must be one of 'sum' or 'mean', got {reduce_fn!r}")
|
|
115
|
+
|
|
116
|
+
return reduce_fn
|
|
69
117
|
|
|
70
118
|
|
|
71
119
|
class ResultEntry(Generic[_OutT]):
|
|
72
120
|
def __init__(
|
|
73
121
|
self,
|
|
74
122
|
meta: Metadata,
|
|
75
|
-
metric: "reax.
|
|
123
|
+
metric: "reax.types.MetricInstance[_OutT]",
|
|
76
124
|
last_value: "reax.types.MetricInstance[_OutT] | None" = None,
|
|
77
125
|
):
|
|
78
126
|
"""Init function."""
|
|
@@ -107,7 +155,7 @@ class ResultCollection(dict[str, ResultEntry]):
|
|
|
107
155
|
self,
|
|
108
156
|
fx: str,
|
|
109
157
|
name: str,
|
|
110
|
-
value: "jt.ArrayLike | reax.
|
|
158
|
+
value: "jt.ArrayLike | reax.types.MetricInstance[_OutT]",
|
|
111
159
|
batch_idx: int,
|
|
112
160
|
*,
|
|
113
161
|
prog_bar: bool = False,
|
|
@@ -115,15 +163,21 @@ class ResultCollection(dict[str, ResultEntry]):
|
|
|
115
163
|
on_step: bool = False,
|
|
116
164
|
on_epoch: bool = True,
|
|
117
165
|
batch_size: int | None = None,
|
|
166
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
118
167
|
):
|
|
119
|
-
"""Log function.
|
|
168
|
+
"""Log function.
|
|
169
|
+
|
|
170
|
+
``reduce_fn`` says how raw values logged over an epoch should be reduced to a single value,
|
|
171
|
+
with ``batch_size`` weighting them when taking their mean. Both are ignored for metric
|
|
172
|
+
instances, which know how to combine themselves.
|
|
173
|
+
"""
|
|
120
174
|
key = f"{fx}.{name}"
|
|
121
175
|
|
|
122
176
|
if isinstance(value, types.MetricInstance):
|
|
123
177
|
metric = value
|
|
124
178
|
else:
|
|
125
179
|
try:
|
|
126
|
-
metric = ArrayResultMetric.create(value, batch_size)
|
|
180
|
+
metric = ArrayResultMetric.create(value, batch_size, reduce_fn)
|
|
127
181
|
except TypeError:
|
|
128
182
|
raise TypeError(
|
|
129
183
|
f"Value must be a `reax.Metric` or a raw value, got {type(value).__name__}"
|
|
@@ -170,7 +170,7 @@ class DataSourceManager(Generic[_T_co]):
|
|
|
170
170
|
) -> "reax.data.DataLoader[_T_co, _U]":
|
|
171
171
|
return self.dataloader.with_new_sampler(sampler)
|
|
172
172
|
|
|
173
|
-
def __init__(self, source: "reax.data.DataSource[_T_co]"):
|
|
173
|
+
def __init__(self, source: "reax.data.DataSource[_T_co, _U]"):
|
|
174
174
|
self._source = source
|
|
175
175
|
self._ready = False
|
|
176
176
|
|
|
@@ -182,7 +182,7 @@ class DataSourceManager(Generic[_T_co]):
|
|
|
182
182
|
return DataSourceManager.LoaderProxy(self, method_name)
|
|
183
183
|
|
|
184
184
|
@property
|
|
185
|
-
def source(self) -> "reax.data.DataSource[_T_co]":
|
|
185
|
+
def source(self) -> "reax.data.DataSource[_T_co, _U]":
|
|
186
186
|
if not self._ready:
|
|
187
187
|
raise RuntimeError(
|
|
188
188
|
"`prepare_and_setup` has not been called, this must be done before accessing the "
|
|
@@ -201,7 +201,7 @@ class DataSourceManager(Generic[_T_co]):
|
|
|
201
201
|
|
|
202
202
|
|
|
203
203
|
def get_datasource(
|
|
204
|
-
datamodule: "reax.DataModule[_T_co] | None" = None,
|
|
204
|
+
datamodule: "reax.DataModule[_T_co, _U] | None" = None,
|
|
205
205
|
module: "reax.Module | None" = None,
|
|
206
206
|
) -> DataSourceManager[_T_co] | None:
|
|
207
207
|
if datamodule is not None:
|
|
@@ -186,7 +186,7 @@ class FitEpoch(train.Train):
|
|
|
186
186
|
|
|
187
187
|
@property
|
|
188
188
|
def val_dataloader(self) -> "reax.DataLoader | None":
|
|
189
|
-
"""
|
|
189
|
+
"""Validation dataloader."""
|
|
190
190
|
if self._validate is None:
|
|
191
191
|
return None
|
|
192
192
|
|
|
@@ -252,18 +252,19 @@ class FitEpoch(train.Train):
|
|
|
252
252
|
def log(
|
|
253
253
|
self,
|
|
254
254
|
name: str,
|
|
255
|
-
value: "jt.ArrayLike | reax.
|
|
255
|
+
value: "jt.ArrayLike | reax.types.MetricInstance",
|
|
256
256
|
batch_size: int | None = None,
|
|
257
257
|
prog_bar: bool = False,
|
|
258
258
|
logger: bool = False,
|
|
259
259
|
on_step=False,
|
|
260
260
|
on_epoch=True,
|
|
261
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
261
262
|
) -> None:
|
|
262
263
|
"""Log function."""
|
|
263
264
|
if self._child is not None:
|
|
264
|
-
self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
|
|
265
|
+
self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
|
|
265
266
|
else:
|
|
266
|
-
super().log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
|
|
267
|
+
super().log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
|
|
267
268
|
|
|
268
269
|
def _should_check_val(self) -> bool:
|
|
269
270
|
"""Decide if we should run validation based on the current mode and progress."""
|
|
@@ -511,6 +512,7 @@ class Fit(stages.Stage):
|
|
|
511
512
|
if not self.enable_validation:
|
|
512
513
|
return None
|
|
513
514
|
|
|
515
|
+
assert self._fit_epoch.validate is not None
|
|
514
516
|
return self._fit_epoch.validate.num_batches
|
|
515
517
|
|
|
516
518
|
@property
|
|
@@ -530,11 +532,12 @@ class Fit(stages.Stage):
|
|
|
530
532
|
|
|
531
533
|
@property
|
|
532
534
|
def train_dataloaders(self) -> "reax.DataLoader":
|
|
535
|
+
"""Training dataloader"""
|
|
533
536
|
return self.train_dataloader
|
|
534
537
|
|
|
535
538
|
@property
|
|
536
539
|
def val_dataloader(self) -> "reax.DataLoader | None":
|
|
537
|
-
"""
|
|
540
|
+
"""Validation dataloader."""
|
|
538
541
|
return self._datamanager.get_dataloader("val")
|
|
539
542
|
|
|
540
543
|
@property
|
|
@@ -555,6 +558,7 @@ class Fit(stages.Stage):
|
|
|
555
558
|
logger: bool = False,
|
|
556
559
|
on_step=False,
|
|
557
560
|
on_epoch=True,
|
|
561
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
558
562
|
) -> None:
|
|
559
563
|
"""Log function."""
|
|
560
564
|
if self._child is None:
|
|
@@ -563,7 +567,7 @@ class Fit(stages.Stage):
|
|
|
563
567
|
"so cannot currently log anything"
|
|
564
568
|
)
|
|
565
569
|
|
|
566
|
-
self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch)
|
|
570
|
+
self._child.log(name, value, batch_size, prog_bar, logger, on_step, on_epoch, reduce_fn)
|
|
567
571
|
|
|
568
572
|
@override
|
|
569
573
|
def _on_iteration_starting(self):
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import TYPE_CHECKING, Any
|
|
1
|
+
from typing import TYPE_CHECKING, Any, Final
|
|
2
2
|
import weakref
|
|
3
3
|
|
|
4
4
|
import beartype
|
|
@@ -35,6 +35,8 @@ class Predict(stages.EpochStage):
|
|
|
35
35
|
fast_dev_run=fast_dev_run,
|
|
36
36
|
limit_batches=limit_batches,
|
|
37
37
|
)
|
|
38
|
+
# Params
|
|
39
|
+
self._mod: "Final[reax.Module]" = module
|
|
38
40
|
self._keep_predictions = keep_predictions
|
|
39
41
|
self._all_outputs: list[Any] | list[list[Any]] | None = []
|
|
40
42
|
|
|
@@ -45,22 +47,22 @@ class Predict(stages.EpochStage):
|
|
|
45
47
|
@override
|
|
46
48
|
def _on_starting(self):
|
|
47
49
|
super()._on_starting()
|
|
48
|
-
self.
|
|
50
|
+
self._mod.on_predict_start(weakref.proxy(self))
|
|
49
51
|
|
|
50
52
|
@override
|
|
51
53
|
def _on_epoch_start(self):
|
|
52
54
|
super()._on_epoch_start()
|
|
53
|
-
self.
|
|
55
|
+
self._mod.on_predict_epoch_start(weakref.proxy(self))
|
|
54
56
|
|
|
55
57
|
@override
|
|
56
58
|
def _on_iteration_starting(self):
|
|
57
59
|
super()._on_iteration_starting()
|
|
58
|
-
self.
|
|
60
|
+
self._mod.on_predict_batch_start(self, self.batch, self.batch_idx)
|
|
59
61
|
|
|
60
62
|
@override
|
|
61
63
|
def _step(self) -> "reax.stages.MetricResults":
|
|
62
64
|
"""Step function."""
|
|
63
|
-
return self.
|
|
65
|
+
return self._mod.predict_step(self.batch, self._iter)
|
|
64
66
|
|
|
65
67
|
@override
|
|
66
68
|
def _on_iteration_finishing(self, outputs: Any, /):
|
|
@@ -69,14 +71,14 @@ class Predict(stages.EpochStage):
|
|
|
69
71
|
if self._keep_predictions:
|
|
70
72
|
cpu = jax.devices("cpu")[0]
|
|
71
73
|
self._all_outputs.append(jax.device_put(outputs, cpu))
|
|
72
|
-
self.
|
|
74
|
+
self._mod.on_predict_batch_end(self, outputs, self.batch, self.batch_idx)
|
|
73
75
|
|
|
74
76
|
@override
|
|
75
77
|
def _on_epoch_end(self) -> None:
|
|
76
78
|
super()._on_epoch_end()
|
|
77
|
-
self.
|
|
79
|
+
self._mod.on_predict_epoch_end(weakref.proxy(self))
|
|
78
80
|
|
|
79
81
|
@override
|
|
80
82
|
def _on_stopped(self) -> None:
|
|
81
83
|
super()._on_stopped()
|
|
82
|
-
self.
|
|
84
|
+
self._mod.on_predict_end(weakref.proxy(self))
|
|
@@ -252,6 +252,7 @@ class Stage(abc.ABC):
|
|
|
252
252
|
logger: bool = False,
|
|
253
253
|
on_step=False,
|
|
254
254
|
on_epoch=True,
|
|
255
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
255
256
|
) -> None:
|
|
256
257
|
"""Log a result while the stage is running."""
|
|
257
258
|
|
|
@@ -391,12 +392,12 @@ class EpochStage(Stage, abc.ABC):
|
|
|
391
392
|
self._outputs = None
|
|
392
393
|
|
|
393
394
|
@property
|
|
394
|
-
def dataloader(self) -> "reax.DataLoader
|
|
395
|
+
def dataloader(self) -> "reax.DataLoader":
|
|
395
396
|
"""Dataloader function."""
|
|
396
397
|
return self._datamanager.get_dataloader(self._dataloader_name)
|
|
397
398
|
|
|
398
399
|
@property
|
|
399
|
-
def dataloaders(self) -> "reax.DataLoader
|
|
400
|
+
def dataloaders(self) -> "reax.DataLoader":
|
|
400
401
|
"""Dataloader function."""
|
|
401
402
|
return self.dataloader
|
|
402
403
|
|
|
@@ -503,6 +504,7 @@ class EpochStage(Stage, abc.ABC):
|
|
|
503
504
|
logger: bool = False,
|
|
504
505
|
on_step=False,
|
|
505
506
|
on_epoch=True,
|
|
507
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
506
508
|
) -> None:
|
|
507
509
|
"""Log metrics during the current epoch."""
|
|
508
510
|
assert self._batch is not None
|
|
@@ -519,6 +521,7 @@ class EpochStage(Stage, abc.ABC):
|
|
|
519
521
|
batch_idx=self._iter,
|
|
520
522
|
on_step=on_step,
|
|
521
523
|
on_epoch=on_epoch,
|
|
524
|
+
reduce_fn=reduce_fn,
|
|
522
525
|
)
|
|
523
526
|
|
|
524
527
|
@override
|
|
@@ -41,27 +41,29 @@ class Test(stages.EpochStage):
|
|
|
41
41
|
fast_dev_run=fast_dev_run,
|
|
42
42
|
limit_batches=limit_batches,
|
|
43
43
|
)
|
|
44
|
+
# Params
|
|
45
|
+
self._mod: "reax.Module" = module
|
|
44
46
|
|
|
45
47
|
@override
|
|
46
48
|
def _on_starting(self):
|
|
47
49
|
super()._on_starting()
|
|
48
|
-
self.
|
|
50
|
+
self._mod.on_test_start(weakref.proxy(self))
|
|
49
51
|
|
|
50
52
|
@override
|
|
51
53
|
def _on_epoch_start(self):
|
|
52
54
|
super()._on_epoch_start()
|
|
53
|
-
self.
|
|
55
|
+
self._mod.on_test_epoch_start(weakref.proxy(self))
|
|
54
56
|
|
|
55
57
|
@override
|
|
56
58
|
def _step(self) -> "reax.stages.MetricResults":
|
|
57
|
-
return self.
|
|
59
|
+
return self._mod.test_step(self.batch, self._iter)
|
|
58
60
|
|
|
59
61
|
@override
|
|
60
62
|
def _on_epoch_end(self):
|
|
61
63
|
super()._on_epoch_end()
|
|
62
|
-
self.
|
|
64
|
+
self._mod.on_test_epoch_end(weakref.proxy(self))
|
|
63
65
|
|
|
64
66
|
@override
|
|
65
67
|
def _on_stopping(self) -> None:
|
|
66
68
|
super()._on_stopping()
|
|
67
|
-
self.
|
|
69
|
+
self._mod.on_test_end(weakref.proxy(self))
|
|
@@ -54,6 +54,7 @@ class Train(stages.EpochStage):
|
|
|
54
54
|
enable_checkpointing=True,
|
|
55
55
|
)
|
|
56
56
|
# Params
|
|
57
|
+
self._mod: "Final[reax.Module]" = module
|
|
57
58
|
self._min_updates: Final[int] = min_updates
|
|
58
59
|
self._max_updates: Final[int | float | None] = max_updates
|
|
59
60
|
self._accumulate_grad_batches: Final[int] = accumulate_grad_batches
|
|
@@ -73,12 +74,12 @@ class Train(stages.EpochStage):
|
|
|
73
74
|
return sum(opt.update_count for opt in self._optimizers)
|
|
74
75
|
|
|
75
76
|
@property
|
|
76
|
-
def optimizers(self) -> list[
|
|
77
|
+
def optimizers(self) -> "list[reax.Optimizer] | None":
|
|
77
78
|
"""Optimizers function."""
|
|
78
79
|
return self._optimizers
|
|
79
80
|
|
|
80
81
|
@override
|
|
81
|
-
def run(self) -> list[
|
|
82
|
+
def run(self) -> "list[reax.Optimizer]":
|
|
82
83
|
"""Run function."""
|
|
83
84
|
super().run()
|
|
84
85
|
return self._optimizers
|
|
@@ -89,14 +90,14 @@ class Train(stages.EpochStage):
|
|
|
89
90
|
super()._on_starting()
|
|
90
91
|
|
|
91
92
|
if not self._optimizers:
|
|
92
|
-
opts = self.
|
|
93
|
+
opts = self._mod.configure_optimizers()
|
|
93
94
|
if opts is None:
|
|
94
95
|
rank_zero.rank_zero_warn(
|
|
95
96
|
"`reax.Module.configure_optimizers` returned `None`, this fit will run "
|
|
96
97
|
"with no optimizer"
|
|
97
98
|
)
|
|
98
99
|
opt = optimizers_.mock_optimizer
|
|
99
|
-
opts = opt, opt.init(self.
|
|
100
|
+
opts = opt, opt.init(self._mod.parameters())
|
|
100
101
|
|
|
101
102
|
if not isinstance(opts, list):
|
|
102
103
|
opts = [opts]
|
|
@@ -107,7 +108,7 @@ class Train(stages.EpochStage):
|
|
|
107
108
|
state = self._engine.to_device(state)
|
|
108
109
|
if self._accumulate_grad_batches > 1:
|
|
109
110
|
stepper = optax.MultiSteps(opt, every_k_schedule=self._accumulate_grad_batches)
|
|
110
|
-
state = stepper.init(self.
|
|
111
|
+
state = stepper.init(self._mod.parameters())
|
|
111
112
|
opt = stepper.gradient_transformation()
|
|
112
113
|
|
|
113
114
|
optimizers.append(optimizers_.Optimizer(opt, state))
|
|
@@ -115,38 +116,38 @@ class Train(stages.EpochStage):
|
|
|
115
116
|
# Create the `Optimizer` instances
|
|
116
117
|
self._optimizers = optimizers
|
|
117
118
|
|
|
118
|
-
self.
|
|
119
|
+
self._mod.on_train_start(weakref.proxy(self))
|
|
119
120
|
|
|
120
121
|
@override
|
|
121
122
|
def _on_iteration_starting(self):
|
|
122
123
|
super()._on_iteration_starting()
|
|
123
|
-
self.
|
|
124
|
+
self._mod.on_train_batch_start(weakref.proxy(self), self.batch, self.batch_idx)
|
|
124
125
|
|
|
125
126
|
@override
|
|
126
127
|
def _on_epoch_start(self):
|
|
127
128
|
"""On started."""
|
|
128
129
|
super()._on_epoch_start()
|
|
129
|
-
self.
|
|
130
|
+
self._mod.on_train_epoch_start(weakref.proxy(self))
|
|
130
131
|
|
|
131
132
|
@override
|
|
132
133
|
def _step(self) -> Any:
|
|
133
134
|
"""Step function."""
|
|
134
|
-
if self.
|
|
135
|
+
if self._mod.parameters() is None:
|
|
135
136
|
raise exceptions.MisconfigurationException(
|
|
136
137
|
"Module does not have any parameters set, this should have been done in "
|
|
137
138
|
".configure_model()."
|
|
138
139
|
)
|
|
139
140
|
|
|
140
|
-
res = self.
|
|
141
|
-
if self.
|
|
141
|
+
res = self._mod.training_step(self.batch, self._iter)
|
|
142
|
+
if self._mod.automatic_optimization:
|
|
142
143
|
if isinstance(res, dict):
|
|
143
144
|
grad = res["grad"]
|
|
144
145
|
loss = res.get("loss", None)
|
|
145
146
|
else:
|
|
146
147
|
loss, grad = res
|
|
147
148
|
opt = self._optimizers[0]
|
|
148
|
-
self.
|
|
149
|
-
opt = opt.update_module(self.
|
|
149
|
+
self._mod.on_before_optimizer_step(opt, grad)
|
|
150
|
+
opt = opt.update_module(self._mod, grad, value=loss)
|
|
150
151
|
self._optimizers = [opt]
|
|
151
152
|
|
|
152
153
|
if (self._min_updates is None or self.updates >= self._min_updates) and (
|
|
@@ -159,17 +160,17 @@ class Train(stages.EpochStage):
|
|
|
159
160
|
@override
|
|
160
161
|
def _on_iteration_finishing(self, outputs: Any, /):
|
|
161
162
|
super()._on_iteration_finishing(outputs)
|
|
162
|
-
self.
|
|
163
|
+
self._mod.on_train_batch_end(weakref.proxy(self), outputs, self.batch, self.batch_idx)
|
|
163
164
|
|
|
164
165
|
@override
|
|
165
166
|
def _on_epoch_end(self) -> None:
|
|
166
167
|
super()._on_epoch_end()
|
|
167
|
-
self.
|
|
168
|
+
self._mod.on_train_epoch_end(weakref.proxy(self))
|
|
168
169
|
|
|
169
170
|
@override
|
|
170
171
|
def _on_stopping(self) -> None:
|
|
171
172
|
super()._on_stopping()
|
|
172
|
-
self.
|
|
173
|
+
self._mod.on_train_end(weakref.proxy(self))
|
|
173
174
|
|
|
174
175
|
@override
|
|
175
176
|
def _done(self) -> bool:
|
|
@@ -192,12 +193,14 @@ class Train(stages.EpochStage):
|
|
|
192
193
|
f"`{type(self).__name__}` stopped: `{type(self).__name__}.should_stop` was set."
|
|
193
194
|
)
|
|
194
195
|
else:
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
f"`min_epochs={self.parent.min_iters!r}` or"
|
|
196
|
+
lines = ["Trainer was signaled to stop but the required "]
|
|
197
|
+
if self.parent is not None:
|
|
198
|
+
lines.append(f"`min_epochs={self.parent.min_iters!r}` or")
|
|
199
|
+
lines.append(
|
|
198
200
|
f" `min_steps={self._min_updates!r}` has not been met. "
|
|
199
201
|
f"Training will continue..."
|
|
200
202
|
)
|
|
203
|
+
self._warning_cache.info("".join(lines))
|
|
201
204
|
return self._stopper.can_stop
|
|
202
205
|
|
|
203
206
|
return False
|
|
@@ -46,6 +46,9 @@ class Validate(stages.EpochStage):
|
|
|
46
46
|
enable_checkpointing=enable_checkpointing,
|
|
47
47
|
)
|
|
48
48
|
|
|
49
|
+
# Params
|
|
50
|
+
self._mod: "reax.Module" = module
|
|
51
|
+
|
|
49
52
|
@property
|
|
50
53
|
def epoch(self) -> int:
|
|
51
54
|
"""Get the current epoch."""
|
|
@@ -54,34 +57,34 @@ class Validate(stages.EpochStage):
|
|
|
54
57
|
@override
|
|
55
58
|
def _on_starting(self):
|
|
56
59
|
super()._on_starting()
|
|
57
|
-
self.
|
|
60
|
+
self._mod.on_validation_start(weakref.proxy(self))
|
|
58
61
|
|
|
59
62
|
@override
|
|
60
63
|
def _on_epoch_start(self):
|
|
61
64
|
super()._on_epoch_start()
|
|
62
|
-
self.
|
|
65
|
+
self._mod.on_validation_epoch_start(weakref.proxy(self))
|
|
63
66
|
|
|
64
67
|
@override
|
|
65
68
|
def _on_iteration_starting(self):
|
|
66
69
|
super()._on_iteration_starting()
|
|
67
|
-
self.
|
|
70
|
+
self._mod.on_validation_batch_start(self, self.batch, self.batch_idx)
|
|
68
71
|
|
|
69
72
|
@override
|
|
70
73
|
def _step(self) -> "reax.stages.MetricResults":
|
|
71
|
-
return self.
|
|
74
|
+
return self._mod.validation_step(self.batch, self._iter)
|
|
72
75
|
|
|
73
76
|
@override
|
|
74
77
|
def _on_iteration_finishing(self, outputs: Any, /):
|
|
75
78
|
"""On iteration finishing."""
|
|
76
79
|
super()._on_iteration_finishing(outputs)
|
|
77
|
-
self.
|
|
80
|
+
self._mod.on_validation_batch_end(self, outputs, self.batch, self.batch_idx)
|
|
78
81
|
|
|
79
82
|
@override
|
|
80
83
|
def _on_epoch_end(self) -> None:
|
|
81
84
|
super()._on_epoch_end()
|
|
82
|
-
self.
|
|
85
|
+
self._mod.on_validation_epoch_end(weakref.proxy(self))
|
|
83
86
|
|
|
84
87
|
@override
|
|
85
88
|
def _on_stopping(self) -> None:
|
|
86
89
|
super()._on_stopping()
|
|
87
|
-
self.
|
|
90
|
+
self._mod.on_validation_end(weakref.proxy(self))
|
|
@@ -230,13 +230,13 @@ class JaxDdpStrategy(_parallel.ParallelStrategy):
|
|
|
230
230
|
multihost_utils.sync_global_devices(name)
|
|
231
231
|
|
|
232
232
|
@override
|
|
233
|
-
def compute(self, metric: "reax.
|
|
233
|
+
def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
|
|
234
234
|
if self.process_count == 1:
|
|
235
235
|
return metric.compute()
|
|
236
236
|
|
|
237
237
|
dynamic, static = eqx.partition(metric, eqx.is_array)
|
|
238
238
|
gathered = multihost_utils.process_allgather(dynamic)
|
|
239
|
-
unbatched: "list[reax.
|
|
239
|
+
unbatched: "list[reax.types.MetricInstance[_OutT]]" = unbatch_pytree(gathered, dynamic)
|
|
240
240
|
# Rejoin with the static data
|
|
241
241
|
unbatched = [eqx.combine(entry, static) for entry in unbatched]
|
|
242
242
|
|
|
@@ -76,5 +76,5 @@ class SingleDevice(_strategies.Strategy):
|
|
|
76
76
|
return None # Nothing to do in a single device
|
|
77
77
|
|
|
78
78
|
@override
|
|
79
|
-
def compute(self, metric: "reax.
|
|
79
|
+
def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
|
|
80
80
|
return metric.compute()
|
|
@@ -123,7 +123,7 @@ class Strategy(abc.ABC):
|
|
|
123
123
|
return data_.ReaxDataLoader(data)
|
|
124
124
|
|
|
125
125
|
@abc.abstractmethod
|
|
126
|
-
def compute(self, metric: "reax.
|
|
126
|
+
def compute(self, metric: "reax.types.MetricInstance[_OutT]") -> _OutT:
|
|
127
127
|
"""Compute the value of a metric, unlike metric.compute(), in a parallel setting this method
|
|
128
128
|
will compute the value across all processes.
|
|
129
129
|
"""
|
|
@@ -395,6 +395,7 @@ class Trainer(stages.StageListener, _deprecated.TrainerDeprecatedMixin):
|
|
|
395
395
|
logger: bool = None,
|
|
396
396
|
on_step=True,
|
|
397
397
|
on_epoch=True,
|
|
398
|
+
reduce_fn: "reax.types.ReduceFn" = "mean",
|
|
398
399
|
) -> None:
|
|
399
400
|
"""Log function."""
|
|
400
401
|
if self._stage is None:
|
|
@@ -412,6 +413,7 @@ class Trainer(stages.StageListener, _deprecated.TrainerDeprecatedMixin):
|
|
|
412
413
|
batch_size=batch_size,
|
|
413
414
|
on_step=on_step,
|
|
414
415
|
on_epoch=on_epoch,
|
|
416
|
+
reduce_fn=reduce_fn,
|
|
415
417
|
)
|
|
416
418
|
|
|
417
419
|
# region Stages
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import pathlib
|
|
2
|
-
from typing import Protocol, TypeVar, runtime_checkable
|
|
2
|
+
from typing import Literal, Protocol, TypeVar, runtime_checkable
|
|
3
3
|
|
|
4
4
|
import jax.typing
|
|
5
5
|
import jaxtyping as jt
|
|
@@ -11,6 +11,10 @@ Path = str | bytes | pathlib.Path
|
|
|
11
11
|
ArrayMask = jt.Int[jt.ArrayLike, "..."] | jt.Bool[jt.ArrayLike, "..."]
|
|
12
12
|
MetricsDict = dict[str, jax.typing.ArrayLike]
|
|
13
13
|
|
|
14
|
+
#: How the raw (i.e. non-metric) values logged over the batches of an epoch should be reduced to a
|
|
15
|
+
#: single value at the end of it: either their mean, or their total.
|
|
16
|
+
ReduceFn = Literal["sum", "mean"]
|
|
17
|
+
|
|
14
18
|
|
|
15
19
|
@runtime_checkable
|
|
16
20
|
class MetricType(Protocol[_OutT]):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|