anemoi-datasets 0.5.16__py3-none-any.whl → 0.5.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anemoi/datasets/__init__.py +4 -1
- anemoi/datasets/__main__.py +12 -2
- anemoi/datasets/_version.py +9 -4
- anemoi/datasets/commands/cleanup.py +17 -2
- anemoi/datasets/commands/compare.py +18 -2
- anemoi/datasets/commands/copy.py +196 -14
- anemoi/datasets/commands/create.py +50 -7
- anemoi/datasets/commands/finalise-additions.py +17 -2
- anemoi/datasets/commands/finalise.py +17 -2
- anemoi/datasets/commands/init-additions.py +17 -2
- anemoi/datasets/commands/init.py +16 -2
- anemoi/datasets/commands/inspect.py +283 -62
- anemoi/datasets/commands/load-additions.py +16 -2
- anemoi/datasets/commands/load.py +16 -2
- anemoi/datasets/commands/patch.py +17 -2
- anemoi/datasets/commands/publish.py +17 -2
- anemoi/datasets/commands/scan.py +31 -3
- anemoi/datasets/compute/recentre.py +47 -11
- anemoi/datasets/create/__init__.py +612 -85
- anemoi/datasets/create/check.py +142 -20
- anemoi/datasets/create/chunks.py +64 -4
- anemoi/datasets/create/config.py +185 -21
- anemoi/datasets/create/filter.py +50 -0
- anemoi/datasets/create/filters/__init__.py +33 -0
- anemoi/datasets/create/filters/empty.py +37 -0
- anemoi/datasets/create/filters/legacy.py +93 -0
- anemoi/datasets/create/filters/noop.py +37 -0
- anemoi/datasets/create/filters/orog_to_z.py +58 -0
- anemoi/datasets/create/{functions/filters → filters}/pressure_level_relative_humidity_to_specific_humidity.py +33 -10
- anemoi/datasets/create/{functions/filters → filters}/pressure_level_specific_humidity_to_relative_humidity.py +32 -8
- anemoi/datasets/create/filters/rename.py +205 -0
- anemoi/datasets/create/{functions/filters → filters}/rotate_winds.py +43 -28
- anemoi/datasets/create/{functions/filters → filters}/single_level_dewpoint_to_relative_humidity.py +32 -9
- anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_dewpoint.py +33 -9
- anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_specific_humidity.py +55 -7
- anemoi/datasets/create/{functions/filters → filters}/single_level_specific_humidity_to_relative_humidity.py +98 -37
- anemoi/datasets/create/filters/speeddir_to_uv.py +95 -0
- anemoi/datasets/create/{functions/filters → filters}/sum.py +24 -27
- anemoi/datasets/create/filters/transform.py +53 -0
- anemoi/datasets/create/{functions/filters → filters}/unrotate_winds.py +27 -18
- anemoi/datasets/create/filters/uv_to_speeddir.py +94 -0
- anemoi/datasets/create/{functions/filters → filters}/wz_to_w.py +51 -33
- anemoi/datasets/create/input/__init__.py +76 -5
- anemoi/datasets/create/input/action.py +149 -13
- anemoi/datasets/create/input/concat.py +81 -10
- anemoi/datasets/create/input/context.py +39 -4
- anemoi/datasets/create/input/data_sources.py +72 -6
- anemoi/datasets/create/input/empty.py +21 -3
- anemoi/datasets/create/input/filter.py +60 -12
- anemoi/datasets/create/input/function.py +154 -37
- anemoi/datasets/create/input/join.py +86 -14
- anemoi/datasets/create/input/misc.py +67 -17
- anemoi/datasets/create/input/pipe.py +33 -6
- anemoi/datasets/create/input/repeated_dates.py +189 -41
- anemoi/datasets/create/input/result.py +202 -87
- anemoi/datasets/create/input/step.py +119 -22
- anemoi/datasets/create/input/template.py +100 -13
- anemoi/datasets/create/input/trace.py +62 -7
- anemoi/datasets/create/patch.py +52 -4
- anemoi/datasets/create/persistent.py +134 -17
- anemoi/datasets/create/size.py +15 -1
- anemoi/datasets/create/source.py +51 -0
- anemoi/datasets/create/sources/__init__.py +36 -0
- anemoi/datasets/create/{functions/sources → sources}/accumulations.py +296 -30
- anemoi/datasets/create/{functions/sources → sources}/constants.py +27 -2
- anemoi/datasets/create/{functions/sources → sources}/eccc_fstd.py +7 -3
- anemoi/datasets/create/sources/empty.py +37 -0
- anemoi/datasets/create/{functions/sources → sources}/forcings.py +25 -1
- anemoi/datasets/create/sources/grib.py +297 -0
- anemoi/datasets/create/{functions/sources → sources}/hindcasts.py +38 -4
- anemoi/datasets/create/sources/legacy.py +93 -0
- anemoi/datasets/create/{functions/sources → sources}/mars.py +168 -20
- anemoi/datasets/create/sources/netcdf.py +42 -0
- anemoi/datasets/create/sources/opendap.py +43 -0
- anemoi/datasets/create/{functions/sources/__init__.py → sources/patterns.py} +35 -4
- anemoi/datasets/create/sources/recentre.py +150 -0
- anemoi/datasets/create/{functions/sources → sources}/source.py +27 -5
- anemoi/datasets/create/{functions/sources → sources}/tendencies.py +64 -7
- anemoi/datasets/create/sources/xarray.py +92 -0
- anemoi/datasets/create/sources/xarray_kerchunk.py +36 -0
- anemoi/datasets/create/sources/xarray_support/README.md +1 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/__init__.py +109 -8
- anemoi/datasets/create/sources/xarray_support/coordinates.py +442 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/field.py +94 -16
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/fieldlist.py +90 -25
- anemoi/datasets/create/sources/xarray_support/flavour.py +1036 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/grid.py +92 -31
- anemoi/datasets/create/sources/xarray_support/metadata.py +395 -0
- anemoi/datasets/create/sources/xarray_support/patch.py +91 -0
- anemoi/datasets/create/sources/xarray_support/time.py +391 -0
- anemoi/datasets/create/sources/xarray_support/variable.py +331 -0
- anemoi/datasets/create/sources/xarray_zarr.py +41 -0
- anemoi/datasets/create/{functions/sources → sources}/zenodo.py +34 -5
- anemoi/datasets/create/statistics/__init__.py +233 -44
- anemoi/datasets/create/statistics/summary.py +52 -6
- anemoi/datasets/create/testing.py +76 -0
- anemoi/datasets/create/{functions/filters/noop.py → typing.py} +6 -3
- anemoi/datasets/create/utils.py +97 -6
- anemoi/datasets/create/writer.py +26 -4
- anemoi/datasets/create/zarr.py +170 -23
- anemoi/datasets/data/__init__.py +51 -4
- anemoi/datasets/data/complement.py +191 -40
- anemoi/datasets/data/concat.py +141 -16
- anemoi/datasets/data/dataset.py +552 -61
- anemoi/datasets/data/debug.py +197 -26
- anemoi/datasets/data/ensemble.py +93 -8
- anemoi/datasets/data/fill_missing.py +165 -18
- anemoi/datasets/data/forwards.py +428 -56
- anemoi/datasets/data/grids.py +323 -97
- anemoi/datasets/data/indexing.py +112 -19
- anemoi/datasets/data/interpolate.py +92 -12
- anemoi/datasets/data/join.py +158 -19
- anemoi/datasets/data/masked.py +129 -15
- anemoi/datasets/data/merge.py +137 -23
- anemoi/datasets/data/misc.py +172 -16
- anemoi/datasets/data/missing.py +233 -29
- anemoi/datasets/data/rescale.py +111 -10
- anemoi/datasets/data/select.py +168 -26
- anemoi/datasets/data/statistics.py +67 -6
- anemoi/datasets/data/stores.py +149 -64
- anemoi/datasets/data/subset.py +159 -25
- anemoi/datasets/data/unchecked.py +168 -57
- anemoi/datasets/data/xy.py +168 -25
- anemoi/datasets/dates/__init__.py +191 -16
- anemoi/datasets/dates/groups.py +189 -47
- anemoi/datasets/grids.py +270 -31
- anemoi/datasets/testing.py +28 -1
- {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/METADATA +9 -6
- anemoi_datasets-0.5.17.dist-info/RECORD +137 -0
- {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/WHEEL +1 -1
- anemoi/datasets/create/functions/__init__.py +0 -66
- anemoi/datasets/create/functions/filters/__init__.py +0 -9
- anemoi/datasets/create/functions/filters/empty.py +0 -17
- anemoi/datasets/create/functions/filters/orog_to_z.py +0 -58
- anemoi/datasets/create/functions/filters/rename.py +0 -79
- anemoi/datasets/create/functions/filters/speeddir_to_uv.py +0 -78
- anemoi/datasets/create/functions/filters/uv_to_speeddir.py +0 -56
- anemoi/datasets/create/functions/sources/empty.py +0 -15
- anemoi/datasets/create/functions/sources/grib.py +0 -150
- anemoi/datasets/create/functions/sources/netcdf.py +0 -15
- anemoi/datasets/create/functions/sources/opendap.py +0 -15
- anemoi/datasets/create/functions/sources/recentre.py +0 -60
- anemoi/datasets/create/functions/sources/xarray/coordinates.py +0 -255
- anemoi/datasets/create/functions/sources/xarray/flavour.py +0 -472
- anemoi/datasets/create/functions/sources/xarray/metadata.py +0 -148
- anemoi/datasets/create/functions/sources/xarray/patch.py +0 -44
- anemoi/datasets/create/functions/sources/xarray/time.py +0 -177
- anemoi/datasets/create/functions/sources/xarray/variable.py +0 -188
- anemoi/datasets/create/functions/sources/xarray_kerchunk.py +0 -42
- anemoi/datasets/create/functions/sources/xarray_zarr.py +0 -15
- anemoi/datasets/utils/fields.py +0 -47
- anemoi_datasets-0.5.16.dist-info/RECORD +0 -129
- {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/entry_points.txt +0 -0
- {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info/licenses}/LICENSE +0 -0
- {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/top_level.txt +0 -0
|
@@ -9,11 +9,19 @@
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
import logging
|
|
12
|
+
from typing import Any
|
|
13
|
+
from typing import Dict
|
|
14
|
+
from typing import Optional
|
|
15
|
+
from typing import Set
|
|
12
16
|
|
|
13
17
|
import numpy as np
|
|
18
|
+
from numpy.typing import NDArray
|
|
14
19
|
|
|
15
20
|
from anemoi.datasets.data import MissingDateError
|
|
16
21
|
|
|
22
|
+
from .dataset import Dataset
|
|
23
|
+
from .dataset import FullIndex
|
|
24
|
+
from .dataset import TupleIndex
|
|
17
25
|
from .debug import Node
|
|
18
26
|
from .debug import debug_indexing
|
|
19
27
|
from .forwards import Forwards
|
|
@@ -26,29 +34,74 @@ LOG = logging.getLogger(__name__)
|
|
|
26
34
|
|
|
27
35
|
|
|
28
36
|
class MissingDatesFill(Forwards):
|
|
29
|
-
|
|
37
|
+
"""Class to handle filling missing dates in a dataset."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, dataset: Dataset) -> None:
|
|
40
|
+
"""Initialize the MissingDatesFill class.
|
|
41
|
+
|
|
42
|
+
Parameters
|
|
43
|
+
----------
|
|
44
|
+
dataset : Dataset
|
|
45
|
+
The dataset with missing dates.
|
|
46
|
+
"""
|
|
30
47
|
super().__init__(dataset)
|
|
31
48
|
self._missing = set(dataset.missing)
|
|
32
|
-
self._warnings = set()
|
|
49
|
+
self._warnings: Set[int] = set()
|
|
33
50
|
|
|
34
51
|
@debug_indexing
|
|
35
52
|
@expand_list_indexing
|
|
36
|
-
def _get_tuple(self, index):
|
|
53
|
+
def _get_tuple(self, index: TupleIndex) -> NDArray[Any]:
|
|
54
|
+
"""Get a tuple of data for the given index.
|
|
55
|
+
|
|
56
|
+
Parameters
|
|
57
|
+
----------
|
|
58
|
+
index : TupleIndex
|
|
59
|
+
The index to retrieve data for.
|
|
60
|
+
|
|
61
|
+
Returns
|
|
62
|
+
-------
|
|
63
|
+
NDArray[Any]
|
|
64
|
+
The data for the given index.
|
|
65
|
+
"""
|
|
37
66
|
index, changes = index_to_slices(index, self.shape)
|
|
38
67
|
index, previous = update_tuple(index, 0, slice(None))
|
|
39
68
|
result = self._get_slice(previous)
|
|
40
69
|
return apply_index_to_slices_changes(result[index], changes)
|
|
41
70
|
|
|
42
|
-
def _get_slice(self, s):
|
|
71
|
+
def _get_slice(self, s: slice) -> NDArray[Any]:
|
|
72
|
+
"""Get a slice of data.
|
|
73
|
+
|
|
74
|
+
Parameters
|
|
75
|
+
----------
|
|
76
|
+
s : slice
|
|
77
|
+
The slice to retrieve data for.
|
|
78
|
+
|
|
79
|
+
Returns
|
|
80
|
+
-------
|
|
81
|
+
NDArray[Any]
|
|
82
|
+
The data for the given slice.
|
|
83
|
+
"""
|
|
43
84
|
return np.stack([self[i] for i in range(*s.indices(self._len))])
|
|
44
85
|
|
|
45
86
|
@property
|
|
46
|
-
def missing(self):
|
|
87
|
+
def missing(self) -> Set[int]:
|
|
88
|
+
"""Get the set of missing dates."""
|
|
47
89
|
return set()
|
|
48
90
|
|
|
49
91
|
@debug_indexing
|
|
50
|
-
def __getitem__(self, n):
|
|
51
|
-
|
|
92
|
+
def __getitem__(self, n: FullIndex) -> NDArray[Any]:
|
|
93
|
+
"""Get the data for the given index.
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
n : FullIndex
|
|
98
|
+
The index to retrieve data for.
|
|
99
|
+
|
|
100
|
+
Returns
|
|
101
|
+
-------
|
|
102
|
+
NDArray[Any]
|
|
103
|
+
The data for the given index.
|
|
104
|
+
"""
|
|
52
105
|
try:
|
|
53
106
|
return self.forward[n]
|
|
54
107
|
except MissingDateError:
|
|
@@ -84,14 +137,39 @@ class MissingDatesFill(Forwards):
|
|
|
84
137
|
|
|
85
138
|
|
|
86
139
|
class MissingDatesClosest(MissingDatesFill):
|
|
87
|
-
|
|
88
|
-
|
|
140
|
+
"""Class to handle filling missing dates using the closest available date."""
|
|
141
|
+
|
|
142
|
+
def __init__(self, dataset: Any, closest: str) -> None:
|
|
143
|
+
"""Initialize the MissingDatesClosest class.
|
|
144
|
+
|
|
145
|
+
Parameters
|
|
146
|
+
----------
|
|
147
|
+
dataset : Any
|
|
148
|
+
The dataset with missing dates.
|
|
149
|
+
closest : str
|
|
150
|
+
The strategy to use for filling missing dates ('up' or 'down').
|
|
151
|
+
"""
|
|
89
152
|
super().__init__(dataset)
|
|
90
153
|
self.closest = closest
|
|
91
154
|
self._closest = {}
|
|
92
155
|
|
|
93
|
-
def _fill_missing(self, n, a, b):
|
|
94
|
-
|
|
156
|
+
def _fill_missing(self, n: int, a: Optional[int], b: Optional[int]) -> NDArray[Any]:
|
|
157
|
+
"""Fill the missing date at the given index.
|
|
158
|
+
|
|
159
|
+
Parameters
|
|
160
|
+
----------
|
|
161
|
+
n : int
|
|
162
|
+
The index of the missing date.
|
|
163
|
+
a : Optional[int]
|
|
164
|
+
The previous available date index.
|
|
165
|
+
b : Optional[int]
|
|
166
|
+
The next available date index.
|
|
167
|
+
|
|
168
|
+
Returns
|
|
169
|
+
-------
|
|
170
|
+
NDArray[Any]
|
|
171
|
+
The filled data for the missing date.
|
|
172
|
+
"""
|
|
95
173
|
if n not in self._warnings:
|
|
96
174
|
LOG.warning(f"Missing date at index {n} ({self.dates[n]})")
|
|
97
175
|
if abs(n - a) == abs(b - n):
|
|
@@ -111,19 +189,58 @@ class MissingDatesClosest(MissingDatesFill):
|
|
|
111
189
|
|
|
112
190
|
return self.forward[self._closest[n]]
|
|
113
191
|
|
|
114
|
-
def
|
|
192
|
+
def forwards_subclass_metadata_specific(self) -> Dict[str, Any]:
|
|
193
|
+
"""Get metadata specific to the subclass.
|
|
194
|
+
|
|
195
|
+
Returns
|
|
196
|
+
-------
|
|
197
|
+
Dict[str, Any]
|
|
198
|
+
The metadata specific to the subclass.
|
|
199
|
+
"""
|
|
115
200
|
return {"closest": self.closest}
|
|
116
201
|
|
|
117
|
-
def tree(self):
|
|
202
|
+
def tree(self) -> Node:
|
|
203
|
+
"""Get the tree representation of the object.
|
|
204
|
+
|
|
205
|
+
Returns
|
|
206
|
+
-------
|
|
207
|
+
Node
|
|
208
|
+
The tree representation of the object.
|
|
209
|
+
"""
|
|
118
210
|
return Node(self, [self.forward.tree()], closest=self.closest)
|
|
119
211
|
|
|
120
212
|
|
|
121
213
|
class MissingDatesInterpolate(MissingDatesFill):
|
|
122
|
-
|
|
214
|
+
"""Class to handle filling missing dates using interpolation."""
|
|
215
|
+
|
|
216
|
+
def __init__(self, dataset: Any) -> None:
|
|
217
|
+
"""Initialize the MissingDatesInterpolate class.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
dataset : Any
|
|
222
|
+
The dataset with missing dates.
|
|
223
|
+
"""
|
|
123
224
|
super().__init__(dataset)
|
|
124
225
|
self._alpha = {}
|
|
125
226
|
|
|
126
|
-
def _fill_missing(self, n, a, b):
|
|
227
|
+
def _fill_missing(self, n: int, a: Optional[int], b: Optional[int]) -> NDArray[Any]:
|
|
228
|
+
"""Fill the missing date at the given index using interpolation.
|
|
229
|
+
|
|
230
|
+
Parameters
|
|
231
|
+
----------
|
|
232
|
+
n : int
|
|
233
|
+
The index of the missing date.
|
|
234
|
+
a : Optional[int]
|
|
235
|
+
The previous available date index.
|
|
236
|
+
b : Optional[int]
|
|
237
|
+
The next available date index.
|
|
238
|
+
|
|
239
|
+
Returns
|
|
240
|
+
-------
|
|
241
|
+
NDArray[Any]
|
|
242
|
+
The filled data for the missing date.
|
|
243
|
+
"""
|
|
127
244
|
if n not in self._warnings:
|
|
128
245
|
LOG.warning(f"Missing date at index {n} ({self.dates[n]})")
|
|
129
246
|
|
|
@@ -147,14 +264,44 @@ class MissingDatesInterpolate(MissingDatesFill):
|
|
|
147
264
|
alpha = self._alpha[n]
|
|
148
265
|
return self.forward[a] * (1 - alpha) + self.forward[b] * alpha
|
|
149
266
|
|
|
150
|
-
def
|
|
267
|
+
def forwards_subclass_metadata_specific(self) -> Dict[str, Any]:
|
|
268
|
+
"""Get metadata specific to the subclass.
|
|
269
|
+
|
|
270
|
+
Returns
|
|
271
|
+
-------
|
|
272
|
+
Dict[str, Any]
|
|
273
|
+
The metadata specific to the subclass.
|
|
274
|
+
"""
|
|
151
275
|
return {}
|
|
152
276
|
|
|
153
|
-
def tree(self):
|
|
277
|
+
def tree(self) -> Node:
|
|
278
|
+
"""Get the tree representation of the object.
|
|
279
|
+
|
|
280
|
+
Returns
|
|
281
|
+
-------
|
|
282
|
+
Node
|
|
283
|
+
The tree representation of the object.
|
|
284
|
+
"""
|
|
154
285
|
return Node(self, [self.forward.tree()])
|
|
155
286
|
|
|
156
287
|
|
|
157
|
-
def fill_missing_dates_factory(dataset, method, kwargs):
|
|
288
|
+
def fill_missing_dates_factory(dataset: Any, method: str, kwargs: Dict[str, Any]) -> Dataset:
|
|
289
|
+
"""Factory function to create an instance of a class to fill missing dates.
|
|
290
|
+
|
|
291
|
+
Parameters
|
|
292
|
+
----------
|
|
293
|
+
dataset : Any
|
|
294
|
+
The dataset with missing dates.
|
|
295
|
+
method : str
|
|
296
|
+
The method to use for filling missing dates ('closest' or 'interpolate').
|
|
297
|
+
kwargs : Dict[str, Any]
|
|
298
|
+
Additional arguments for the method.
|
|
299
|
+
|
|
300
|
+
Returns
|
|
301
|
+
-------
|
|
302
|
+
Dataset
|
|
303
|
+
An instance of a class to fill missing dates.
|
|
304
|
+
"""
|
|
158
305
|
if method == "closest":
|
|
159
306
|
closest = kwargs.get("closest", "up")
|
|
160
307
|
return MissingDatesClosest(dataset, closest=closest)
|