reax 0.7.0.dev2__tar.gz → 0.7.0.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/PKG-INFO +1 -1
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/pyproject.toml +4 -2
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/__init__.py +1 -1
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/_loaders.py +46 -24
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/_types.py +7 -3
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/samplers.py +74 -64
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/model_checkpoint.py +11 -1
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/_classification.py +1 -1
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/_metric.py +3 -3
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/collections.py +1 -1
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/stats.py +15 -4
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_parallel.py +4 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/LICENSE.txt +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/README.rst +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/_engine.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/_module_hooks.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/_datasource_manager.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/_datasources.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/collate.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/dataloaders.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/datamodules.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/datasets.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/fetchers.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/kfold.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/data/utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/demos/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/demos/boring_classes.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/demos/mnist.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/exceptions.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/hooks.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/keys.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/lightning/LICENSE +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/lightning/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/lightning/rank_zero.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/checkpointer.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/early_stopping.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/progress/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/progress/progress_bar.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/progress/rich_progress.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/progress/tqdm_progress.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/listeners/utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/_utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/csv_logs.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/logger.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/mlflow.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/pandas.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/loggers/tensorboard.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/_evaluators.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/_globals.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/_registry.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/aggregation.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/jm/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/jm/types.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/jm/utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/regression.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/metrics/utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/modules.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/optimizers.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/plugins.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/profilers/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/profilers/jax_profiler.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/profilers/profiler.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/random.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/results.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/saving.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/_loops.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/_timer.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/common.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/fit.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/predict.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/stages.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/test.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/train.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/stages/validation.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_factory.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_jax.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_single_device.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_strategies.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/strategies/_utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/testing.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/_checkpoint_connector.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/_checkpointing.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/_hooks_map.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/_logger_connector.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/_utils.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/training/trainer.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/types.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/utils/__init__.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/utils/arrays.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/utils/containers.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/utils/events.py +0 -0
- {reax-0.7.0.dev2 → reax-0.7.0.dev3}/src/reax/utils/grads.py +0 -0
|
@@ -147,13 +147,11 @@ exclude = [
|
|
|
147
147
|
'docs/source/get_started/quick_start.ipynb',
|
|
148
148
|
'examples/autoencoder.py',
|
|
149
149
|
'src/reax/data/_datasource_manager.py',
|
|
150
|
-
'src/reax/data/_loaders.py',
|
|
151
150
|
'src/reax/data/_types.py',
|
|
152
151
|
'src/reax/data/collate.py',
|
|
153
152
|
'src/reax/data/datamodules.py',
|
|
154
153
|
'src/reax/data/datasets.py',
|
|
155
154
|
'src/reax/data/fetchers.py',
|
|
156
|
-
'src/reax/data/samplers.py',
|
|
157
155
|
'src/reax/data/utils.py',
|
|
158
156
|
'src/reax/demos/boring_classes.py',
|
|
159
157
|
'src/reax/demos/mnist.py',
|
|
@@ -210,6 +208,10 @@ exclude = [
|
|
|
210
208
|
'tests/utils/test_containers.py',
|
|
211
209
|
]
|
|
212
210
|
|
|
211
|
+
[tool.ty.analysis]
|
|
212
|
+
allowed-unresolved-imports = ["torch.**"]
|
|
213
|
+
|
|
214
|
+
|
|
213
215
|
[tool.pylint.MASTER]
|
|
214
216
|
ignore-paths = '^src/reax/lightning/.*$' # These are copy-paste from lightning, so don't lint
|
|
215
217
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
from collections.abc import
|
|
2
|
-
from typing import TYPE_CHECKING, Final, TypeVar
|
|
1
|
+
from collections.abc import Iterator
|
|
2
|
+
from typing import TYPE_CHECKING, Any, Final, TypeVar
|
|
3
3
|
|
|
4
4
|
import beartype
|
|
5
5
|
import jax
|
|
@@ -18,7 +18,8 @@ __all__ = "ArrayLoader", "CachingLoader", "ReaxDataLoader", "FetcherDataLoader",
|
|
|
18
18
|
T = TypeVar("T")
|
|
19
19
|
_U = TypeVar("_U")
|
|
20
20
|
_T_co = TypeVar("_T_co", covariant=True)
|
|
21
|
-
|
|
21
|
+
Array = jax.Array | np.ndarray
|
|
22
|
+
ArrayOrArrayTuple = Array | tuple[Array, ...]
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
def _single_or_value(value: tuple[T, ...], to_test=None) -> T | tuple[T, ...]:
|
|
@@ -37,7 +38,7 @@ class FetcherDataLoader(_types.DataLoader[_T_co, _U]):
|
|
|
37
38
|
def __init__(
|
|
38
39
|
self,
|
|
39
40
|
dataset: "reax.data.Dataset[_T_co]",
|
|
40
|
-
sampler: _types.Sampler
|
|
41
|
+
sampler: _types.Sampler,
|
|
41
42
|
collate_fn: _types.CollateFn,
|
|
42
43
|
):
|
|
43
44
|
# Params
|
|
@@ -54,7 +55,7 @@ class FetcherDataLoader(_types.DataLoader[_T_co, _U]):
|
|
|
54
55
|
|
|
55
56
|
@override
|
|
56
57
|
@property
|
|
57
|
-
def sampler(self):
|
|
58
|
+
def sampler(self) -> "reax.data.Sampler":
|
|
58
59
|
return self._sampler
|
|
59
60
|
|
|
60
61
|
def __iter__(self) -> Iterator[_U]:
|
|
@@ -83,7 +84,7 @@ class ReaxDataLoader(FetcherDataLoader[_T_co, _U]):
|
|
|
83
84
|
dataset: "reax.data.Dataset[_T_co]",
|
|
84
85
|
batch_size: int = 1,
|
|
85
86
|
shuffle: bool = False,
|
|
86
|
-
sampler=None,
|
|
87
|
+
sampler: "reax.data.Sampler | None" = None,
|
|
87
88
|
collate_fn: _types.CollateFn | None = None,
|
|
88
89
|
):
|
|
89
90
|
sampler = samplers.create_sampler(
|
|
@@ -96,7 +97,7 @@ class ReaxDataLoader(FetcherDataLoader[_T_co, _U]):
|
|
|
96
97
|
super().__init__(dataset, sampler, collate_fn)
|
|
97
98
|
|
|
98
99
|
|
|
99
|
-
class ArrayLoader(_types.DataLoader[
|
|
100
|
+
class ArrayLoader(_types.DataLoader[Any, ArrayOrArrayTuple]):
|
|
100
101
|
"""A more efficient loader for array datasets that uses .take() to batch over slices of an
|
|
101
102
|
array dataset.
|
|
102
103
|
"""
|
|
@@ -106,12 +107,24 @@ class ArrayLoader(_types.DataLoader[ArrayOrArrayTuple, ArrayOrArrayTuple]):
|
|
|
106
107
|
self,
|
|
107
108
|
arrays: ArrayOrArrayTuple,
|
|
108
109
|
batch_size: int = 1,
|
|
109
|
-
shuffle=False,
|
|
110
|
+
shuffle: bool = False,
|
|
110
111
|
sampler: "reax.data.Sampler | None" = None,
|
|
112
|
+
*,
|
|
113
|
+
_batched_sampler: "reax.data.Sampler | None" = None,
|
|
111
114
|
):
|
|
115
|
+
"""Init function.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
arrays: The array, or tuple of equal-length arrays, to load from.
|
|
119
|
+
batch_size: Number of rows per batch.
|
|
120
|
+
shuffle: Whether to shuffle the rows.
|
|
121
|
+
sampler: Optional per-row index sampler. It is batched using ``batch_size``.
|
|
122
|
+
_batched_sampler: Internal, used by :meth:`with_new_sampler`. An already batched
|
|
123
|
+
sampler that is used as-is.
|
|
124
|
+
"""
|
|
112
125
|
# Params
|
|
113
|
-
self._batch_size: int = batch_size
|
|
114
|
-
self._shuffle: bool = shuffle
|
|
126
|
+
self._batch_size: Final[int] = batch_size
|
|
127
|
+
self._shuffle: Final[bool] = shuffle
|
|
115
128
|
|
|
116
129
|
if isinstance(arrays, tuple):
|
|
117
130
|
if not all(arrays[0].shape[0] == array.shape[0] for array in arrays):
|
|
@@ -123,24 +136,26 @@ class ArrayLoader(_types.DataLoader[ArrayOrArrayTuple, ArrayOrArrayTuple]):
|
|
|
123
136
|
first_array = arrays
|
|
124
137
|
|
|
125
138
|
# State
|
|
126
|
-
self._arrays = arrays
|
|
127
|
-
self._sampler
|
|
128
|
-
|
|
129
|
-
|
|
139
|
+
self._arrays: Final[ArrayOrArrayTuple] = arrays
|
|
140
|
+
self._sampler: reax.data.Sampler
|
|
141
|
+
if _batched_sampler is not None:
|
|
142
|
+
self._sampler = _batched_sampler
|
|
143
|
+
else:
|
|
144
|
+
self._sampler = samplers.create_sampler(
|
|
145
|
+
first_array, batch_size=batch_size, shuffle=shuffle, sampler=sampler
|
|
146
|
+
)
|
|
130
147
|
|
|
131
148
|
def __iter__(self) -> Iterator[ArrayOrArrayTuple]:
|
|
132
149
|
"""Iter function."""
|
|
133
|
-
for
|
|
134
|
-
|
|
150
|
+
for indices in self._sampler:
|
|
151
|
+
if len(indices) == 0:
|
|
152
|
+
continue # Padding entry from a DistributedSampler
|
|
153
|
+
idx = np.asarray(indices, dtype=np.intp)
|
|
135
154
|
if isinstance(self._arrays, tuple):
|
|
136
155
|
yield tuple(array.take(idx, axis=0) for array in self._arrays)
|
|
137
156
|
else:
|
|
138
157
|
yield self._arrays.take(idx, axis=0)
|
|
139
158
|
|
|
140
|
-
def __len__(self) -> int:
|
|
141
|
-
"""Len function."""
|
|
142
|
-
return len(self._sampler)
|
|
143
|
-
|
|
144
159
|
@property
|
|
145
160
|
def batch_size(self) -> int:
|
|
146
161
|
return self._batch_size
|
|
@@ -149,21 +164,28 @@ class ArrayLoader(_types.DataLoader[ArrayOrArrayTuple, ArrayOrArrayTuple]):
|
|
|
149
164
|
def shuffle(self) -> bool:
|
|
150
165
|
return self._shuffle
|
|
151
166
|
|
|
167
|
+
@override
|
|
152
168
|
@property
|
|
153
169
|
def dataset(self) -> ArrayOrArrayTuple:
|
|
154
170
|
return self._arrays
|
|
155
171
|
|
|
172
|
+
@override
|
|
156
173
|
@property
|
|
157
|
-
def sampler(self):
|
|
174
|
+
def sampler(self) -> "reax.data.Sampler":
|
|
158
175
|
return self._sampler
|
|
159
176
|
|
|
160
177
|
def first(self) -> ArrayOrArrayTuple:
|
|
161
178
|
"""First function."""
|
|
162
179
|
return next(iter(self))
|
|
163
180
|
|
|
181
|
+
@override
|
|
164
182
|
def with_new_sampler(self, sampler: "reax.data.Sampler") -> "ArrayLoader":
|
|
183
|
+
# Like the other loaders, ``sampler`` replaces ``self.sampler`` (which is already batched)
|
|
165
184
|
return ArrayLoader(
|
|
166
|
-
arrays=self.
|
|
185
|
+
arrays=self._arrays,
|
|
186
|
+
batch_size=self._batch_size,
|
|
187
|
+
shuffle=self._shuffle,
|
|
188
|
+
_batched_sampler=sampler,
|
|
167
189
|
)
|
|
168
190
|
|
|
169
191
|
|
|
@@ -178,7 +200,7 @@ class CachingLoader(_types.DataLoader[_T_co, _U]):
|
|
|
178
200
|
self._loader = loader
|
|
179
201
|
self._reset_every = reset_every
|
|
180
202
|
self._time_since_reset = 0
|
|
181
|
-
self._cache = None
|
|
203
|
+
self._cache: list[_U] | None = None
|
|
182
204
|
|
|
183
205
|
def __iter__(self) -> Iterator[_U]:
|
|
184
206
|
"""Iter function."""
|
|
@@ -205,7 +227,7 @@ class CachingLoader(_types.DataLoader[_T_co, _U]):
|
|
|
205
227
|
|
|
206
228
|
@override
|
|
207
229
|
@property
|
|
208
|
-
def sampler(self):
|
|
230
|
+
def sampler(self) -> "reax.data.Sampler":
|
|
209
231
|
return self._loader.sampler
|
|
210
232
|
|
|
211
233
|
@override
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import abc
|
|
2
|
-
from collections.abc import Callable, Iterable, Sequence
|
|
2
|
+
from collections.abc import Callable, Iterable, Sequence, Sized
|
|
3
3
|
from typing import TYPE_CHECKING, Generic, TypeVar
|
|
4
4
|
|
|
5
5
|
if TYPE_CHECKING:
|
|
6
6
|
import reax
|
|
7
7
|
|
|
8
|
-
__all__ = "Sampler", "DataLoader", "Dataset", "CollateFn"
|
|
8
|
+
__all__ = "Sampler", "DataLoader", "Dataset", "CollateFn", "BatchSampler"
|
|
9
9
|
|
|
10
10
|
_T_co = TypeVar("_T_co", covariant=True)
|
|
11
11
|
U = TypeVar("U")
|
|
@@ -13,6 +13,7 @@ IdxT = TypeVar("IdxT")
|
|
|
13
13
|
|
|
14
14
|
Dataset = Iterable[_T_co] | Sequence[_T_co]
|
|
15
15
|
Sampler = Iterable[IdxT]
|
|
16
|
+
BatchSampler = Sampler[list[IdxT]]
|
|
16
17
|
CollateFn = Callable[[Sequence[_T_co]], U]
|
|
17
18
|
|
|
18
19
|
|
|
@@ -25,7 +26,10 @@ class DataLoader(Generic[_T_co, U], Iterable[U], abc.ABC):
|
|
|
25
26
|
"""
|
|
26
27
|
|
|
27
28
|
def __len__(self) -> int:
|
|
28
|
-
|
|
29
|
+
sampler = self.sampler
|
|
30
|
+
if not isinstance(sampler, Sized):
|
|
31
|
+
raise TypeError(f"Sampler {type(sampler).__name__} has no length.")
|
|
32
|
+
return len(sampler)
|
|
29
33
|
|
|
30
34
|
@property
|
|
31
35
|
@abc.abstractmethod
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
from collections.abc import
|
|
1
|
+
from collections.abc import Iterable, Iterator, Sequence, Sized
|
|
2
2
|
import contextlib
|
|
3
3
|
import functools
|
|
4
4
|
import itertools
|
|
5
5
|
import math
|
|
6
|
-
from typing import TYPE_CHECKING, Final, TypeVar
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar
|
|
7
7
|
|
|
8
8
|
import beartype
|
|
9
9
|
import jax
|
|
@@ -15,20 +15,35 @@ from . import _types
|
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
17
|
import reax
|
|
18
|
+
import reax.data
|
|
18
19
|
|
|
19
20
|
__all__ = (
|
|
20
21
|
"SequentialSampler",
|
|
21
22
|
"RandomSampler",
|
|
22
|
-
"
|
|
23
|
+
"ChunkedSampler",
|
|
23
24
|
"IterableSampler",
|
|
24
25
|
"DistributedSampler",
|
|
25
26
|
)
|
|
26
27
|
|
|
27
28
|
_T_co = TypeVar("_T_co", covariant=True)
|
|
28
|
-
_IdxT = TypeVar("_IdxT"
|
|
29
|
+
_IdxT = TypeVar("_IdxT")
|
|
29
30
|
Empty = list[Never]
|
|
30
31
|
|
|
31
32
|
|
|
33
|
+
def _try_len(obj: object) -> int | None:
|
|
34
|
+
"""Return ``len(obj)``, or ``None`` if it has no (usable) length.
|
|
35
|
+
|
|
36
|
+
Wrappers such as :class:`ChunkedSampler` always define ``__len__`` but raise ``TypeError`` when
|
|
37
|
+
the wrapped sampler is unsized, so an ``isinstance(obj, Sized)`` check isn't enough.
|
|
38
|
+
"""
|
|
39
|
+
if not isinstance(obj, Sized):
|
|
40
|
+
return None
|
|
41
|
+
try:
|
|
42
|
+
return len(obj)
|
|
43
|
+
except TypeError:
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
32
47
|
class SequentialSampler(_types.Sampler[int]):
|
|
33
48
|
"""Sequentially sample integers index samples up to a given `length`.
|
|
34
49
|
|
|
@@ -80,17 +95,19 @@ class RandomSampler(_types.Sampler[int]):
|
|
|
80
95
|
|
|
81
96
|
if self._replacements:
|
|
82
97
|
for _ in range(self.num_samples // self.SAMPLE_SIZE):
|
|
83
|
-
yield from np.random.randint(0, high=total, size=(self.SAMPLE_SIZE,))
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
98
|
+
yield from map(int, np.random.randint(0, high=total, size=(self.SAMPLE_SIZE,)))
|
|
99
|
+
|
|
100
|
+
yield from map(
|
|
101
|
+
int, np.random.randint(0, high=total, size=(self.num_samples % self.SAMPLE_SIZE,))
|
|
102
|
+
)
|
|
87
103
|
else:
|
|
88
104
|
for _ in range(self.num_samples // total):
|
|
89
|
-
yield from np.random.permutation(total)
|
|
90
|
-
|
|
105
|
+
yield from map(int, np.random.permutation(total))
|
|
106
|
+
|
|
107
|
+
yield from map(int, np.random.permutation(total)[: self.num_samples % total])
|
|
91
108
|
|
|
92
109
|
|
|
93
|
-
class
|
|
110
|
+
class ChunkedSampler(Generic[_IdxT], _types.Sampler[list[_IdxT]]):
|
|
94
111
|
r"""Sample batches of indexes from a given sample."""
|
|
95
112
|
|
|
96
113
|
def __init__(self, sampler: _types.Sampler[_IdxT], batch_size: int, drop_last: bool) -> None:
|
|
@@ -113,29 +130,23 @@ class BatchSampler(_types.Sampler[list[_IdxT]]):
|
|
|
113
130
|
|
|
114
131
|
def __iter__(self) -> Iterator[list[_IdxT]]:
|
|
115
132
|
"""Iter function."""
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
else:
|
|
125
|
-
batch = [0] * self._batch_size
|
|
126
|
-
idx_in_batch = 0
|
|
127
|
-
for idx in self._sampler:
|
|
128
|
-
batch[idx_in_batch] = idx
|
|
129
|
-
idx_in_batch += 1
|
|
130
|
-
if idx_in_batch == self._batch_size:
|
|
131
|
-
yield batch
|
|
132
|
-
idx_in_batch = 0
|
|
133
|
-
batch = [0] * self._batch_size
|
|
134
|
-
if idx_in_batch > 0:
|
|
135
|
-
yield batch[:idx_in_batch]
|
|
133
|
+
sampler_iter = iter(self._sampler)
|
|
134
|
+
while True:
|
|
135
|
+
batch = list(itertools.islice(sampler_iter, self._batch_size))
|
|
136
|
+
if not batch:
|
|
137
|
+
return
|
|
138
|
+
if self._drop_last and len(batch) < self._batch_size:
|
|
139
|
+
return
|
|
140
|
+
yield batch
|
|
136
141
|
|
|
137
142
|
def __len__(self) -> int:
|
|
138
|
-
"""Len function.
|
|
143
|
+
"""Len function.
|
|
144
|
+
|
|
145
|
+
Raises:
|
|
146
|
+
TypeError: If the wrapped sampler is unsized.
|
|
147
|
+
"""
|
|
148
|
+
if not isinstance(self._sampler, Sized):
|
|
149
|
+
raise TypeError(f"Wrapped sampler {type(self._sampler).__name__} has no length.")
|
|
139
150
|
if self._drop_last:
|
|
140
151
|
return len(self._sampler) // self._batch_size
|
|
141
152
|
|
|
@@ -143,12 +154,18 @@ class BatchSampler(_types.Sampler[list[_IdxT]]):
|
|
|
143
154
|
|
|
144
155
|
|
|
145
156
|
class IterableSampler(_types.Sampler[list[None]]):
|
|
157
|
+
"""Endless sampler for iterable (non random-access) datasets.
|
|
158
|
+
|
|
159
|
+
Each item is a placeholder "index" ``[None]``: the iterable fetcher ignores the values and just
|
|
160
|
+
pulls one element from the dataset per entry, stopping when the dataset is exhausted.
|
|
161
|
+
"""
|
|
162
|
+
|
|
146
163
|
def __iter__(self) -> Iterator[list[None]]:
|
|
147
164
|
"""Iter function."""
|
|
148
165
|
yield from itertools.repeat([None])
|
|
149
166
|
|
|
150
167
|
|
|
151
|
-
class DistributedSampler(_types.Sampler[_T_co]):
|
|
168
|
+
class DistributedSampler(_types.Sampler[_T_co | Empty]):
|
|
152
169
|
"""A sampler that distributes data across multiple processes.
|
|
153
170
|
|
|
154
171
|
This sampler is a pure sharding wrapper around an inner sampler (e.g.,
|
|
@@ -192,10 +209,7 @@ class DistributedSampler(_types.Sampler[_T_co]):
|
|
|
192
209
|
self._drop_last: Final[bool] = drop_last
|
|
193
210
|
|
|
194
211
|
# Sized inner: we know the global step count arithmetically.
|
|
195
|
-
|
|
196
|
-
self._global_steps: int | None = len(inner)
|
|
197
|
-
except TypeError:
|
|
198
|
-
self._global_steps: int | None = None
|
|
212
|
+
self._global_steps: Final[int | None] = _try_len(inner)
|
|
199
213
|
|
|
200
214
|
@staticmethod
|
|
201
215
|
def _init_num_replicas(num_replicas: int | None) -> int:
|
|
@@ -260,28 +274,34 @@ def create_sampler(
|
|
|
260
274
|
batch_size: int | None = None,
|
|
261
275
|
replacements: bool = False,
|
|
262
276
|
shuffle: bool = False,
|
|
263
|
-
sampler: "reax.data.Sampler[
|
|
277
|
+
sampler: "reax.data.Sampler[Any] | None" = None,
|
|
264
278
|
drop_last: bool = False,
|
|
265
|
-
) -> "reax.data.Sampler[
|
|
266
|
-
"""Create sampler.
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
279
|
+
) -> "reax.data.Sampler[Any] | reax.data.BatchSampler[Any]":
|
|
280
|
+
"""Create a sampler for ``dataset``, batching it if ``batch_size`` is given.
|
|
281
|
+
|
|
282
|
+
If ``sampler`` is passed it is used as-is (and batched if requested), otherwise a default
|
|
283
|
+
sampler is created based on the dataset type.
|
|
284
|
+
"""
|
|
285
|
+
base: reax.data.Sampler[Any]
|
|
286
|
+
if sampler is not None:
|
|
287
|
+
base = sampler
|
|
288
|
+
elif isinstance(dataset, jax.Array):
|
|
289
|
+
# Need this special case because jax arrays are not really subclasses of jax.Array and
|
|
290
|
+
# won't be matched by singledispatch
|
|
291
|
+
base = create_sequence_sampler(dataset, replacements=replacements, shuffle=shuffle)
|
|
292
|
+
else:
|
|
293
|
+
base = _create_sampler(dataset, replacements=replacements, shuffle=shuffle)
|
|
274
294
|
|
|
275
|
-
if batch_size is
|
|
276
|
-
|
|
295
|
+
if batch_size is None:
|
|
296
|
+
return base
|
|
277
297
|
|
|
278
|
-
return
|
|
298
|
+
return ChunkedSampler(base, batch_size, drop_last)
|
|
279
299
|
|
|
280
300
|
|
|
281
301
|
@functools.singledispatch
|
|
282
302
|
def _create_sampler(
|
|
283
303
|
dataset: _types.Dataset[_T_co], replacements: bool = False, shuffle: bool = False
|
|
284
|
-
) -> "reax.data.Sampler[
|
|
304
|
+
) -> "reax.data.Sampler[Any]":
|
|
285
305
|
"""Create sampler."""
|
|
286
306
|
raise TypeError(f"Unsupported type {type(dataset).__name__}")
|
|
287
307
|
|
|
@@ -310,7 +330,7 @@ with contextlib.suppress(ImportError):
|
|
|
310
330
|
@_create_sampler.register(np.ndarray)
|
|
311
331
|
def create_sequence_sampler(
|
|
312
332
|
dataset, replacements: bool = False, shuffle: bool = False
|
|
313
|
-
) -> "reax.data.Sampler[
|
|
333
|
+
) -> "reax.data.Sampler[int]":
|
|
314
334
|
"""Create sequence sampler."""
|
|
315
335
|
if shuffle:
|
|
316
336
|
return RandomSampler(len(dataset), replacements=replacements)
|
|
@@ -320,8 +340,8 @@ def create_sequence_sampler(
|
|
|
320
340
|
|
|
321
341
|
@_create_sampler.register(Iterable)
|
|
322
342
|
def create_iterable_sampler(
|
|
323
|
-
dataset: Iterable[
|
|
324
|
-
) ->
|
|
343
|
+
dataset: Iterable[Any], replacements: bool = False, shuffle: bool = False
|
|
344
|
+
) -> IterableSampler:
|
|
325
345
|
"""Create iterable sampler."""
|
|
326
346
|
if shuffle:
|
|
327
347
|
raise ValueError(
|
|
@@ -335,13 +355,3 @@ def create_iterable_sampler(
|
|
|
335
355
|
)
|
|
336
356
|
|
|
337
357
|
return IterableSampler()
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
def create_batch_sampler(
|
|
341
|
-
dataset: Sequence[_T_co], replacements: bool = False, shuffle: bool = False
|
|
342
|
-
) -> BatchSampler[int]:
|
|
343
|
-
"""Create batch sampler."""
|
|
344
|
-
if shuffle:
|
|
345
|
-
return RandomSampler(len(dataset), replacements=replacements)
|
|
346
|
-
|
|
347
|
-
return SequentialSampler(len(dataset))
|
|
@@ -313,6 +313,8 @@ class ModelCheckpoint(checkpointer.Checkpointer):
|
|
|
313
313
|
self._best_model_score: jax.Array | None = None
|
|
314
314
|
# Keep track of the global number of optimizer steps
|
|
315
315
|
self._last_global_step_saved = 0
|
|
316
|
+
# De-duplicate warnings so the same message is only emitted once
|
|
317
|
+
self._warning_cache = rank_zero.WarningCache()
|
|
316
318
|
|
|
317
319
|
@staticmethod
|
|
318
320
|
def __init_triggers(
|
|
@@ -579,8 +581,16 @@ class ModelCheckpoint(checkpointer.Checkpointer):
|
|
|
579
581
|
assert self._monitor is not None
|
|
580
582
|
current = monitor_candidates.get(self._monitor)
|
|
581
583
|
|
|
584
|
+
if current is None:
|
|
585
|
+
self._warning_cache.warn(
|
|
586
|
+
f"The monitored key {self._monitor!r} was not found in the metrics available at "
|
|
587
|
+
f"checkpoint time, so no 'best' checkpoint can be saved. Make sure the metric is "
|
|
588
|
+
f"logged before the checkpoint listener runs; when monitoring a validation "
|
|
589
|
+
f"metric you may need `ModelCheckpoint(..., save_on_train_epoch_end=False)`."
|
|
590
|
+
)
|
|
591
|
+
return
|
|
592
|
+
|
|
582
593
|
if self.check_monitor_top_k(current):
|
|
583
|
-
assert current is not None
|
|
584
594
|
self._update_best_and_save(current, trainer, monitor_candidates)
|
|
585
595
|
elif self._verbose:
|
|
586
596
|
epoch = monitor_candidates["epoch"]
|
|
@@ -76,13 +76,13 @@ class Metric(equinox.Module, Generic[_OutT], metaclass=abc.ABCMeta):
|
|
|
76
76
|
return FromFunction
|
|
77
77
|
|
|
78
78
|
@hybrid_method
|
|
79
|
-
def empty(
|
|
79
|
+
def empty(self) -> "Metric[_OutT]": # pylint: disable=no-self-argument
|
|
80
80
|
"""Create a new empty instance.
|
|
81
81
|
|
|
82
82
|
This can be called on the class or an instance. By default, this will call the constructor
|
|
83
83
|
with no arguments, if needed, subclasses can overwrite this with custom behaviour.
|
|
84
84
|
"""
|
|
85
|
-
cls =
|
|
85
|
+
cls = self if isinstance(self, type) else type(self)
|
|
86
86
|
return cls()
|
|
87
87
|
|
|
88
88
|
@hybrid_method
|
|
@@ -154,7 +154,7 @@ class FromFun(Metric[_OutT]):
|
|
|
154
154
|
|
|
155
155
|
@override
|
|
156
156
|
@hybrid_method
|
|
157
|
-
def empty(self
|
|
157
|
+
def empty(self) -> "FromFun[_OutT]":
|
|
158
158
|
"""Empty function."""
|
|
159
159
|
if isinstance(self, type):
|
|
160
160
|
cls = cast("type[FromFun[_OutT]]", self)
|
|
@@ -21,7 +21,7 @@ MetricType = type[metric_.Metric] | metric_.Metric
|
|
|
21
21
|
class MetricCollection(equinox.Module):
|
|
22
22
|
"""A collection of metrics that can be created/updated/merged using a single call."""
|
|
23
23
|
|
|
24
|
-
_metrics: flax.core.FrozenDict[str,
|
|
24
|
+
_metrics: "flax.core.FrozenDict[str, reax.Metric]"
|
|
25
25
|
_evaluator: "reax.metrics.MetricEvaluator"
|
|
26
26
|
|
|
27
27
|
@jt.jaxtyped(typechecker=beartype.beartype)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Stage for evaluating dataset statistics"""
|
|
2
2
|
|
|
3
|
-
from typing import TYPE_CHECKING, Any
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
|
4
4
|
|
|
5
5
|
import beartype
|
|
6
6
|
import jaxtyping as jt
|
|
@@ -16,7 +16,18 @@ if TYPE_CHECKING:
|
|
|
16
16
|
import reax
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
__all__ = ("EvaluateStats", "StatsEpoch")
|
|
19
|
+
__all__ = ("EvaluateStats", "StatsEpoch", "StatsStage")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@runtime_checkable
|
|
23
|
+
class StatsStage(_loops.HostWithData, Protocol):
|
|
24
|
+
"""Structural interface a stage must provide to run a :class:`StatsEpoch`.
|
|
25
|
+
|
|
26
|
+
``StatsEpoch`` is generic over this protocol rather than a concrete stage, so
|
|
27
|
+
statistics can be computed on any stage that exposes the data/loop interface (for
|
|
28
|
+
example a custom stage such as ``FromData``), without forcing it to inherit from
|
|
29
|
+
:class:`EvaluateStats`.
|
|
30
|
+
"""
|
|
20
31
|
|
|
21
32
|
|
|
22
33
|
class EvaluateStats(stages.Stage):
|
|
@@ -63,10 +74,10 @@ class EvaluateStats(stages.Stage):
|
|
|
63
74
|
return self.run_loop(self._loop)
|
|
64
75
|
|
|
65
76
|
|
|
66
|
-
class StatsEpoch(_loops.LoopOverBatches[
|
|
77
|
+
class StatsEpoch(_loops.LoopOverBatches[StatsStage]):
|
|
67
78
|
def __init__(
|
|
68
79
|
self,
|
|
69
|
-
stage:
|
|
80
|
+
stage: StatsStage,
|
|
70
81
|
*,
|
|
71
82
|
stats: "reax.Metric | Sequence[reax.Metric] | dict[str, reax.Metric]",
|
|
72
83
|
fast_dev_run: bool | int = False,
|
|
@@ -79,4 +79,8 @@ class ParallelDataLoader(data_.DataLoader[_T_co, _U]):
|
|
|
79
79
|
pass
|
|
80
80
|
|
|
81
81
|
def with_new_sampler(self, sampler: "reax.data.Sampler") -> "ParallelDataLoader[_T_co, _U]":
|
|
82
|
+
if not isinstance(sampler, data_.samplers.DistributedSampler):
|
|
83
|
+
raise TypeError(
|
|
84
|
+
f"ParallelDataLoader requires a DistributedSampler, got {type(sampler).__name__}"
|
|
85
|
+
)
|
|
82
86
|
return ParallelDataLoader(dataset=self._dataset, sampler=sampler, fetcher=self._fetcher)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|