anemoi-datasets 0.5.25__py3-none-any.whl → 0.5.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anemoi/datasets/__init__.py +1 -2
- anemoi/datasets/_version.py +16 -3
- anemoi/datasets/commands/check.py +1 -1
- anemoi/datasets/commands/copy.py +1 -2
- anemoi/datasets/commands/create.py +1 -1
- anemoi/datasets/commands/grib-index.py +1 -1
- anemoi/datasets/commands/inspect.py +27 -35
- anemoi/datasets/commands/validate.py +59 -0
- anemoi/datasets/compute/recentre.py +3 -6
- anemoi/datasets/create/__init__.py +22 -25
- anemoi/datasets/create/check.py +10 -12
- anemoi/datasets/create/chunks.py +1 -2
- anemoi/datasets/create/config.py +3 -6
- anemoi/datasets/create/filter.py +21 -24
- anemoi/datasets/create/input/__init__.py +1 -2
- anemoi/datasets/create/input/action.py +3 -5
- anemoi/datasets/create/input/concat.py +5 -8
- anemoi/datasets/create/input/context.py +3 -6
- anemoi/datasets/create/input/data_sources.py +5 -8
- anemoi/datasets/create/input/empty.py +1 -2
- anemoi/datasets/create/input/filter.py +2 -3
- anemoi/datasets/create/input/function.py +1 -2
- anemoi/datasets/create/input/join.py +4 -5
- anemoi/datasets/create/input/misc.py +4 -6
- anemoi/datasets/create/input/repeated_dates.py +13 -18
- anemoi/datasets/create/input/result.py +29 -33
- anemoi/datasets/create/input/step.py +6 -24
- anemoi/datasets/create/input/template.py +3 -4
- anemoi/datasets/create/input/trace.py +1 -1
- anemoi/datasets/create/patch.py +1 -2
- anemoi/datasets/create/persistent.py +3 -5
- anemoi/datasets/create/size.py +1 -3
- anemoi/datasets/create/sources/accumulations.py +47 -52
- anemoi/datasets/create/sources/accumulations2.py +4 -8
- anemoi/datasets/create/sources/constants.py +1 -3
- anemoi/datasets/create/sources/empty.py +1 -2
- anemoi/datasets/create/sources/fdb.py +133 -0
- anemoi/datasets/create/sources/forcings.py +1 -2
- anemoi/datasets/create/sources/grib.py +6 -10
- anemoi/datasets/create/sources/grib_index.py +13 -15
- anemoi/datasets/create/sources/hindcasts.py +2 -5
- anemoi/datasets/create/sources/legacy.py +1 -1
- anemoi/datasets/create/sources/mars.py +17 -21
- anemoi/datasets/create/sources/netcdf.py +1 -2
- anemoi/datasets/create/sources/opendap.py +1 -3
- anemoi/datasets/create/sources/patterns.py +4 -6
- anemoi/datasets/create/sources/planetary_computer.py +44 -0
- anemoi/datasets/create/sources/recentre.py +8 -11
- anemoi/datasets/create/sources/source.py +3 -6
- anemoi/datasets/create/sources/tendencies.py +2 -5
- anemoi/datasets/create/sources/xarray.py +4 -6
- anemoi/datasets/create/sources/xarray_support/__init__.py +15 -32
- anemoi/datasets/create/sources/xarray_support/coordinates.py +16 -12
- anemoi/datasets/create/sources/xarray_support/field.py +17 -16
- anemoi/datasets/create/sources/xarray_support/fieldlist.py +11 -15
- anemoi/datasets/create/sources/xarray_support/flavour.py +83 -45
- anemoi/datasets/create/sources/xarray_support/grid.py +15 -9
- anemoi/datasets/create/sources/xarray_support/metadata.py +19 -128
- anemoi/datasets/create/sources/xarray_support/patch.py +47 -6
- anemoi/datasets/create/sources/xarray_support/time.py +10 -13
- anemoi/datasets/create/sources/xarray_support/variable.py +27 -23
- anemoi/datasets/create/sources/xarray_zarr.py +1 -2
- anemoi/datasets/create/sources/zenodo.py +3 -5
- anemoi/datasets/create/statistics/__init__.py +3 -6
- anemoi/datasets/create/testing.py +2 -74
- anemoi/datasets/create/typing.py +1 -2
- anemoi/datasets/create/utils.py +1 -2
- anemoi/datasets/create/zarr.py +7 -2
- anemoi/datasets/data/__init__.py +15 -6
- anemoi/datasets/data/complement.py +52 -23
- anemoi/datasets/data/concat.py +5 -8
- anemoi/datasets/data/dataset.py +42 -47
- anemoi/datasets/data/debug.py +7 -9
- anemoi/datasets/data/ensemble.py +4 -6
- anemoi/datasets/data/fill_missing.py +7 -10
- anemoi/datasets/data/forwards.py +30 -28
- anemoi/datasets/data/grids.py +12 -16
- anemoi/datasets/data/indexing.py +9 -12
- anemoi/datasets/data/interpolate.py +7 -15
- anemoi/datasets/data/join.py +8 -12
- anemoi/datasets/data/masked.py +6 -11
- anemoi/datasets/data/merge.py +5 -9
- anemoi/datasets/data/misc.py +41 -45
- anemoi/datasets/data/missing.py +11 -16
- anemoi/datasets/data/observations/__init__.py +8 -14
- anemoi/datasets/data/padded.py +3 -5
- anemoi/datasets/data/records/backends/__init__.py +2 -2
- anemoi/datasets/data/rescale.py +5 -12
- anemoi/datasets/data/select.py +13 -16
- anemoi/datasets/data/statistics.py +4 -7
- anemoi/datasets/data/stores.py +23 -77
- anemoi/datasets/data/subset.py +8 -11
- anemoi/datasets/data/unchecked.py +7 -11
- anemoi/datasets/data/xy.py +25 -21
- anemoi/datasets/dates/__init__.py +13 -18
- anemoi/datasets/dates/groups.py +7 -10
- anemoi/datasets/grids.py +11 -12
- anemoi/datasets/testing.py +93 -7
- anemoi/datasets/validate.py +598 -0
- {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/METADATA +5 -4
- anemoi_datasets-0.5.27.dist-info/RECORD +134 -0
- anemoi/datasets/create/filters/__init__.py +0 -33
- anemoi/datasets/create/filters/empty.py +0 -37
- anemoi/datasets/create/filters/legacy.py +0 -93
- anemoi/datasets/create/filters/noop.py +0 -37
- anemoi/datasets/create/filters/orog_to_z.py +0 -58
- anemoi/datasets/create/filters/pressure_level_relative_humidity_to_specific_humidity.py +0 -83
- anemoi/datasets/create/filters/pressure_level_specific_humidity_to_relative_humidity.py +0 -84
- anemoi/datasets/create/filters/rename.py +0 -205
- anemoi/datasets/create/filters/rotate_winds.py +0 -105
- anemoi/datasets/create/filters/single_level_dewpoint_to_relative_humidity.py +0 -78
- anemoi/datasets/create/filters/single_level_relative_humidity_to_dewpoint.py +0 -84
- anemoi/datasets/create/filters/single_level_relative_humidity_to_specific_humidity.py +0 -163
- anemoi/datasets/create/filters/single_level_specific_humidity_to_relative_humidity.py +0 -451
- anemoi/datasets/create/filters/speeddir_to_uv.py +0 -95
- anemoi/datasets/create/filters/sum.py +0 -68
- anemoi/datasets/create/filters/transform.py +0 -51
- anemoi/datasets/create/filters/unrotate_winds.py +0 -105
- anemoi/datasets/create/filters/uv_to_speeddir.py +0 -94
- anemoi/datasets/create/filters/wz_to_w.py +0 -98
- anemoi/datasets/utils/__init__.py +0 -8
- anemoi_datasets-0.5.25.dist-info/RECORD +0 -150
- {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/WHEEL +0 -0
- {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/entry_points.txt +0 -0
- {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/licenses/LICENSE +0 -0
- {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/top_level.txt +0 -0
|
@@ -15,10 +15,6 @@ from collections import defaultdict
|
|
|
15
15
|
from functools import cached_property
|
|
16
16
|
from typing import Any
|
|
17
17
|
from typing import DefaultDict
|
|
18
|
-
from typing import Dict
|
|
19
|
-
from typing import List
|
|
20
|
-
from typing import Optional
|
|
21
|
-
from typing import Tuple
|
|
22
18
|
|
|
23
19
|
import numpy as np
|
|
24
20
|
from anemoi.utils.dates import as_timedelta
|
|
@@ -33,7 +29,7 @@ from .trace import trace_datasource
|
|
|
33
29
|
LOG = logging.getLogger(__name__)
|
|
34
30
|
|
|
35
31
|
|
|
36
|
-
def _fields_metatata(variables:
|
|
32
|
+
def _fields_metatata(variables: tuple[str, ...], cube: Any) -> dict[str, Any]:
|
|
37
33
|
"""Retrieve metadata for the given variables and cube.
|
|
38
34
|
|
|
39
35
|
Parameters
|
|
@@ -50,7 +46,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
50
46
|
"""
|
|
51
47
|
assert isinstance(variables, tuple), variables
|
|
52
48
|
|
|
53
|
-
KNOWN:
|
|
49
|
+
KNOWN: dict[str, dict[str, bool]] = {
|
|
54
50
|
"cos_julian_day": dict(computed_forcing=True, constant_in_time=False),
|
|
55
51
|
"cos_latitude": dict(computed_forcing=True, constant_in_time=True),
|
|
56
52
|
"cos_local_time": dict(computed_forcing=True, constant_in_time=False),
|
|
@@ -65,9 +61,9 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
65
61
|
"sin_longitude": dict(computed_forcing=True, constant_in_time=True),
|
|
66
62
|
}
|
|
67
63
|
|
|
68
|
-
def _merge(md1:
|
|
64
|
+
def _merge(md1: dict[str, Any], md2: dict[str, Any]) -> dict[str, Any]:
|
|
69
65
|
assert set(md1.keys()) == set(md2.keys()), (set(md1.keys()), set(md2.keys()))
|
|
70
|
-
result:
|
|
66
|
+
result: dict[str, Any] = {}
|
|
71
67
|
for k in md1.keys():
|
|
72
68
|
v1 = md1[k]
|
|
73
69
|
v2 = md2[k]
|
|
@@ -90,10 +86,10 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
90
86
|
|
|
91
87
|
return result
|
|
92
88
|
|
|
93
|
-
mars:
|
|
94
|
-
other: DefaultDict[str,
|
|
89
|
+
mars: dict[str, Any] = {}
|
|
90
|
+
other: DefaultDict[str, dict[str, Any]] = defaultdict(dict)
|
|
95
91
|
i: int = -1
|
|
96
|
-
date:
|
|
92
|
+
date: str | None = None
|
|
97
93
|
for c in cube.iterate_cubelets():
|
|
98
94
|
|
|
99
95
|
if date is None:
|
|
@@ -137,7 +133,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
137
133
|
|
|
138
134
|
if startStep != endStep:
|
|
139
135
|
# https://codes.ecmwf.int/grib/format/grib2/ctables/4/10/
|
|
140
|
-
TYPE_OF_STATISTICAL_PROCESSING:
|
|
136
|
+
TYPE_OF_STATISTICAL_PROCESSING: dict[int | None, str | None] = {
|
|
141
137
|
None: None,
|
|
142
138
|
0: "average",
|
|
143
139
|
1: "accumulation",
|
|
@@ -157,12 +153,12 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
157
153
|
|
|
158
154
|
# https://codes.ecmwf.int/grib/format/grib1/ctable/5/
|
|
159
155
|
|
|
160
|
-
TIME_RANGE_INDICATOR:
|
|
156
|
+
TIME_RANGE_INDICATOR: dict[int, str] = {
|
|
161
157
|
4: "accumulation",
|
|
162
158
|
3: "average",
|
|
163
159
|
}
|
|
164
160
|
|
|
165
|
-
STEP_TYPE_FOR_CONVERSION:
|
|
161
|
+
STEP_TYPE_FOR_CONVERSION: dict[str, str] = {
|
|
166
162
|
"min": "minimum",
|
|
167
163
|
"max": "maximum",
|
|
168
164
|
"accum": "accumulation",
|
|
@@ -172,7 +168,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
172
168
|
# A few patches
|
|
173
169
|
#
|
|
174
170
|
|
|
175
|
-
PATCHES:
|
|
171
|
+
PATCHES: dict[str, str] = {
|
|
176
172
|
"10fg6": "maximum",
|
|
177
173
|
"mntpr3": "minimum", # Not in param db
|
|
178
174
|
"mntpr6": "minimum", # Not in param db
|
|
@@ -211,7 +207,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
211
207
|
else:
|
|
212
208
|
mars[variables[i]] = md
|
|
213
209
|
|
|
214
|
-
result:
|
|
210
|
+
result: dict[str, dict[str, Any]] = {}
|
|
215
211
|
for k, v in mars.items():
|
|
216
212
|
result[k] = dict(mars=v) if v else {}
|
|
217
213
|
result[k].update(other[k])
|
|
@@ -222,7 +218,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
|
|
|
222
218
|
return result
|
|
223
219
|
|
|
224
220
|
|
|
225
|
-
def _data_request(data: Any) ->
|
|
221
|
+
def _data_request(data: Any) -> dict[str, Any]:
|
|
226
222
|
"""Build a data request dictionary from the given data.
|
|
227
223
|
|
|
228
224
|
Parameters
|
|
@@ -235,12 +231,12 @@ def _data_request(data: Any) -> Dict[str, Any]:
|
|
|
235
231
|
dict
|
|
236
232
|
The data request dictionary.
|
|
237
233
|
"""
|
|
238
|
-
date:
|
|
234
|
+
date: Any | None = None
|
|
239
235
|
params_levels: DefaultDict[str, set] = defaultdict(set)
|
|
240
236
|
params_steps: DefaultDict[str, set] = defaultdict(set)
|
|
241
237
|
|
|
242
|
-
area:
|
|
243
|
-
grid:
|
|
238
|
+
area: Any | None = None
|
|
239
|
+
grid: Any | None = None
|
|
244
240
|
|
|
245
241
|
for field in data:
|
|
246
242
|
try:
|
|
@@ -270,8 +266,8 @@ def _data_request(data: Any) -> Dict[str, Any]:
|
|
|
270
266
|
except Exception:
|
|
271
267
|
LOG.error(f"Error in retrieving metadata (cannot build data request info) for {field}", exc_info=True)
|
|
272
268
|
|
|
273
|
-
def sort(old_dic: DefaultDict[str, set]) ->
|
|
274
|
-
new_dic:
|
|
269
|
+
def sort(old_dic: DefaultDict[str, set]) -> dict[str, list[Any]]:
|
|
270
|
+
new_dic: dict[str, list[Any]] = {}
|
|
275
271
|
for k, v in old_dic.items():
|
|
276
272
|
new_dic[k] = sorted(list(v))
|
|
277
273
|
return new_dic
|
|
@@ -288,7 +284,7 @@ class Result:
|
|
|
288
284
|
empty: bool = False
|
|
289
285
|
_coords_already_built: bool = False
|
|
290
286
|
|
|
291
|
-
def __init__(self, context: ActionContext, action_path:
|
|
287
|
+
def __init__(self, context: ActionContext, action_path: list[str], dates: Any) -> None:
|
|
292
288
|
"""Initialize a Result instance.
|
|
293
289
|
|
|
294
290
|
Parameters
|
|
@@ -309,7 +305,7 @@ class Result:
|
|
|
309
305
|
|
|
310
306
|
self.context: Any = context
|
|
311
307
|
self.group_of_dates: Any = dates
|
|
312
|
-
self.action_path:
|
|
308
|
+
self.action_path: list[str] = action_path
|
|
313
309
|
|
|
314
310
|
@property
|
|
315
311
|
@trace_datasource
|
|
@@ -318,7 +314,7 @@ class Result:
|
|
|
318
314
|
self._raise_not_implemented()
|
|
319
315
|
|
|
320
316
|
@property
|
|
321
|
-
def data_request(self) ->
|
|
317
|
+
def data_request(self) -> dict[str, Any]:
|
|
322
318
|
"""Returns a dictionary with the parameters needed to retrieve the data."""
|
|
323
319
|
return _data_request(self.datasource)
|
|
324
320
|
|
|
@@ -340,7 +336,7 @@ class Result:
|
|
|
340
336
|
LOG.debug("Sorting dataset %s %s", dict(order_by), remapping)
|
|
341
337
|
assert order_by, order_by
|
|
342
338
|
|
|
343
|
-
patches:
|
|
339
|
+
patches: dict[str, dict[Any | None, int]] = {"number": {None: 0}}
|
|
344
340
|
|
|
345
341
|
try:
|
|
346
342
|
cube: Any = ds.cube(
|
|
@@ -377,7 +373,7 @@ class Result:
|
|
|
377
373
|
patches : Any
|
|
378
374
|
The patches configuration.
|
|
379
375
|
"""
|
|
380
|
-
METADATA:
|
|
376
|
+
METADATA: tuple[str, ...] = (
|
|
381
377
|
"date",
|
|
382
378
|
"time",
|
|
383
379
|
"step",
|
|
@@ -402,7 +398,7 @@ class Result:
|
|
|
402
398
|
# print("Executing", self.action_path)
|
|
403
399
|
# print("Dates:", compress_dates(self.dates))
|
|
404
400
|
|
|
405
|
-
names:
|
|
401
|
+
names: list[str] = []
|
|
406
402
|
for a in args:
|
|
407
403
|
if isinstance(a, str):
|
|
408
404
|
names.append(a)
|
|
@@ -421,7 +417,7 @@ class Result:
|
|
|
421
417
|
print(" ", n)
|
|
422
418
|
|
|
423
419
|
print()
|
|
424
|
-
user_shape:
|
|
420
|
+
user_shape: tuple[int, ...] = tuple(len(v) for k, v in user_coords.items())
|
|
425
421
|
print("Shape of the hypercube :", user_shape)
|
|
426
422
|
print(
|
|
427
423
|
"Number of expected fields :", math.prod(user_shape), "=", " x ".join([str(i) for i in user_shape])
|
|
@@ -643,13 +639,13 @@ class Result:
|
|
|
643
639
|
self._coords_already_built: bool = True
|
|
644
640
|
|
|
645
641
|
@property
|
|
646
|
-
def variables(self) ->
|
|
642
|
+
def variables(self) -> list[str]:
|
|
647
643
|
"""Retrieve the variables for the result."""
|
|
648
644
|
self.build_coords()
|
|
649
645
|
return self._variables
|
|
650
646
|
|
|
651
647
|
@property
|
|
652
|
-
def variables_metadata(self) ->
|
|
648
|
+
def variables_metadata(self) -> dict[str, Any]:
|
|
653
649
|
"""Retrieve the metadata for the variables."""
|
|
654
650
|
return _fields_metatata(self.variables, self._cube)
|
|
655
651
|
|
|
@@ -690,7 +686,7 @@ class Result:
|
|
|
690
686
|
return self._proj_string
|
|
691
687
|
|
|
692
688
|
@cached_property
|
|
693
|
-
def shape(self) ->
|
|
689
|
+
def shape(self) -> list[int]:
|
|
694
690
|
"""Retrieve the shape of the result."""
|
|
695
691
|
return [
|
|
696
692
|
len(self.group_of_dates),
|
|
@@ -700,7 +696,7 @@ class Result:
|
|
|
700
696
|
]
|
|
701
697
|
|
|
702
698
|
@cached_property
|
|
703
|
-
def coords(self) ->
|
|
699
|
+
def coords(self) -> dict[str, Any]:
|
|
704
700
|
"""Retrieve the coordinates of the result."""
|
|
705
701
|
return {
|
|
706
702
|
"dates": list(self.group_of_dates),
|
|
@@ -8,13 +8,8 @@
|
|
|
8
8
|
# nor does it submit to any jurisdiction.
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
|
-
import warnings
|
|
12
11
|
from copy import deepcopy
|
|
13
12
|
from typing import Any
|
|
14
|
-
from typing import Dict
|
|
15
|
-
from typing import List
|
|
16
|
-
from typing import Optional
|
|
17
|
-
from typing import Type
|
|
18
13
|
|
|
19
14
|
from .action import Action
|
|
20
15
|
from .action import ActionContext
|
|
@@ -31,7 +26,7 @@ class StepResult(Result):
|
|
|
31
26
|
"""Represents the result of a step in the data processing pipeline."""
|
|
32
27
|
|
|
33
28
|
def __init__(
|
|
34
|
-
self, context: Context, action_path:
|
|
29
|
+
self, context: Context, action_path: list[str], group_of_dates: Any, action: Action, upstream_result: Result
|
|
35
30
|
) -> None:
|
|
36
31
|
"""Initialize a StepResult instance.
|
|
37
32
|
|
|
@@ -64,10 +59,10 @@ class StepResult(Result):
|
|
|
64
59
|
class StepAction(Action):
|
|
65
60
|
"""Represents an action that is part of a step in the data processing pipeline."""
|
|
66
61
|
|
|
67
|
-
result_class:
|
|
62
|
+
result_class: type[StepResult] | None = None
|
|
68
63
|
|
|
69
64
|
def __init__(
|
|
70
|
-
self, context: ActionContext, action_path:
|
|
65
|
+
self, context: ActionContext, action_path: list[str], previous_step: Any, *args: Any, **kwargs: Any
|
|
71
66
|
) -> None:
|
|
72
67
|
"""Initialize a StepAction instance.
|
|
73
68
|
|
|
@@ -116,7 +111,7 @@ class StepAction(Action):
|
|
|
116
111
|
return self._repr(self.previous_step, _inline_=str(self.kwargs))
|
|
117
112
|
|
|
118
113
|
|
|
119
|
-
def step_factory(config:
|
|
114
|
+
def step_factory(config: dict[str, Any], context: ActionContext, action_path: list[str], previous_step: Any) -> Any:
|
|
120
115
|
"""Factory function to create a step action based on the given configuration.
|
|
121
116
|
|
|
122
117
|
Parameters
|
|
@@ -165,24 +160,11 @@ def step_factory(config: Dict[str, Any], context: ActionContext, action_path: Li
|
|
|
165
160
|
if cls is not None:
|
|
166
161
|
return cls(context, action_path, previous_step, *args, **kwargs)
|
|
167
162
|
|
|
168
|
-
# Try filters from
|
|
163
|
+
# Try filters from transform filter registry
|
|
169
164
|
from anemoi.transform.filters import filter_registry as transform_filter_registry
|
|
170
165
|
|
|
171
|
-
from ..filters import create_filter as create_datasets_filter
|
|
172
|
-
from ..filters import filter_registry as datasets_filter_registry
|
|
173
|
-
|
|
174
|
-
if datasets_filter_registry.is_registered(key):
|
|
175
|
-
|
|
176
|
-
if transform_filter_registry.is_registered(key):
|
|
177
|
-
warnings.warn(f"Filter `{key}` is registered in both datasets and transform filter registries")
|
|
178
|
-
|
|
179
|
-
filter = create_datasets_filter(None, config)
|
|
180
|
-
return FunctionStepAction(context, action_path + [key], previous_step, key, filter)
|
|
181
|
-
|
|
182
|
-
# Use filters from transform registry
|
|
183
|
-
|
|
184
166
|
if transform_filter_registry.is_registered(key):
|
|
185
|
-
from ..
|
|
167
|
+
from ..filter import TransformFilter
|
|
186
168
|
|
|
187
169
|
return FunctionStepAction(
|
|
188
170
|
context, action_path + [key], previous_step, key, TransformFilter(context, key, config)
|
|
@@ -12,10 +12,9 @@ import logging
|
|
|
12
12
|
import re
|
|
13
13
|
from abc import ABC
|
|
14
14
|
from abc import abstractmethod
|
|
15
|
+
from collections.abc import Callable
|
|
15
16
|
from functools import wraps
|
|
16
17
|
from typing import Any
|
|
17
|
-
from typing import Callable
|
|
18
|
-
from typing import List
|
|
19
18
|
|
|
20
19
|
from .context import Context
|
|
21
20
|
|
|
@@ -68,7 +67,7 @@ class Substitution(ABC):
|
|
|
68
67
|
class Reference(Substitution):
|
|
69
68
|
"""A class to represent a reference to another value in the context."""
|
|
70
69
|
|
|
71
|
-
def __init__(self, context: Any, action_path:
|
|
70
|
+
def __init__(self, context: Any, action_path: list[str]) -> None:
|
|
72
71
|
"""Initialize a Reference instance.
|
|
73
72
|
|
|
74
73
|
Parameters
|
|
@@ -79,7 +78,7 @@ class Reference(Substitution):
|
|
|
79
78
|
The action path to resolve.
|
|
80
79
|
"""
|
|
81
80
|
self.context: Any = context
|
|
82
|
-
self.action_path:
|
|
81
|
+
self.action_path: list[str] = action_path
|
|
83
82
|
|
|
84
83
|
def resolve(self, context: Context) -> Any:
|
|
85
84
|
"""Resolve the reference using the given context.
|
anemoi/datasets/create/patch.py
CHANGED
|
@@ -10,14 +10,13 @@
|
|
|
10
10
|
import json
|
|
11
11
|
import logging
|
|
12
12
|
import os
|
|
13
|
-
from typing import Union
|
|
14
13
|
|
|
15
14
|
import zarr
|
|
16
15
|
|
|
17
16
|
LOG = logging.getLogger(__name__)
|
|
18
17
|
|
|
19
18
|
|
|
20
|
-
def fix_order_by(order_by:
|
|
19
|
+
def fix_order_by(order_by: dict | list) -> list[dict]:
|
|
21
20
|
"""Fix the order_by attribute to ensure it is a list of dictionaries.
|
|
22
21
|
|
|
23
22
|
Parameters
|
|
@@ -16,9 +16,8 @@ import os
|
|
|
16
16
|
import pickle
|
|
17
17
|
import shutil
|
|
18
18
|
import socket
|
|
19
|
+
from collections.abc import Iterator
|
|
19
20
|
from typing import Any
|
|
20
|
-
from typing import Iterator
|
|
21
|
-
from typing import Tuple
|
|
22
21
|
|
|
23
22
|
import numpy as np
|
|
24
23
|
from anemoi.utils.provenance import gather_provenance_info
|
|
@@ -198,7 +197,7 @@ class BufferedPersistentDict(PersistentDict):
|
|
|
198
197
|
self.elements = []
|
|
199
198
|
self.keys = []
|
|
200
199
|
|
|
201
|
-
def items(self) -> Iterator[
|
|
200
|
+
def items(self) -> Iterator[tuple[Any, Any]]:
|
|
202
201
|
"""Yield items stored in the BufferedPersistentDict.
|
|
203
202
|
|
|
204
203
|
Yields
|
|
@@ -207,8 +206,7 @@ class BufferedPersistentDict(PersistentDict):
|
|
|
207
206
|
An iterator over the items.
|
|
208
207
|
"""
|
|
209
208
|
for keys, elements in self.storage.items():
|
|
210
|
-
|
|
211
|
-
yield key, elt
|
|
209
|
+
yield from zip(keys, elements)
|
|
212
210
|
|
|
213
211
|
def delete(self) -> None:
|
|
214
212
|
"""Delete the storage directory and its contents."""
|
anemoi/datasets/create/size.py
CHANGED
|
@@ -10,8 +10,6 @@
|
|
|
10
10
|
|
|
11
11
|
import logging
|
|
12
12
|
import os
|
|
13
|
-
from typing import Dict
|
|
14
|
-
from typing import Optional
|
|
15
13
|
|
|
16
14
|
import tqdm
|
|
17
15
|
from anemoi.utils.humanize import bytes_to_human
|
|
@@ -19,7 +17,7 @@ from anemoi.utils.humanize import bytes_to_human
|
|
|
19
17
|
LOG = logging.getLogger(__name__)
|
|
20
18
|
|
|
21
19
|
|
|
22
|
-
def compute_directory_sizes(path: str) ->
|
|
20
|
+
def compute_directory_sizes(path: str) -> dict[str, int] | None:
|
|
23
21
|
"""Computes the total size and number of files in a directory.
|
|
24
22
|
|
|
25
23
|
Parameters
|
|
@@ -10,14 +10,9 @@
|
|
|
10
10
|
import datetime
|
|
11
11
|
import logging
|
|
12
12
|
import warnings
|
|
13
|
+
from collections.abc import Generator
|
|
13
14
|
from copy import deepcopy
|
|
14
15
|
from typing import Any
|
|
15
|
-
from typing import Dict
|
|
16
|
-
from typing import Generator
|
|
17
|
-
from typing import List
|
|
18
|
-
from typing import Optional
|
|
19
|
-
from typing import Tuple
|
|
20
|
-
from typing import Union
|
|
21
16
|
|
|
22
17
|
import earthkit.data as ekd
|
|
23
18
|
import numpy as np
|
|
@@ -66,10 +61,10 @@ class Accumulation:
|
|
|
66
61
|
date: int,
|
|
67
62
|
time: int,
|
|
68
63
|
number: int,
|
|
69
|
-
step:
|
|
64
|
+
step: list[int],
|
|
70
65
|
frequency: int,
|
|
71
|
-
accumulations_reset_frequency:
|
|
72
|
-
user_date:
|
|
66
|
+
accumulations_reset_frequency: int | None = None,
|
|
67
|
+
user_date: str | None = None,
|
|
73
68
|
**kwargs: Any,
|
|
74
69
|
) -> None:
|
|
75
70
|
"""Initialises an Accumulation instance.
|
|
@@ -103,10 +98,10 @@ class Accumulation:
|
|
|
103
98
|
self.time = time
|
|
104
99
|
self.steps = step
|
|
105
100
|
self.number = number
|
|
106
|
-
self.values:
|
|
101
|
+
self.values: NDArray[None] | None = None
|
|
107
102
|
self.seen = set()
|
|
108
|
-
self.startStep:
|
|
109
|
-
self.endStep:
|
|
103
|
+
self.startStep: int | None = None
|
|
104
|
+
self.endStep: int | None = None
|
|
110
105
|
self.done = False
|
|
111
106
|
self.frequency = frequency
|
|
112
107
|
self.accumulations_reset_frequency = accumulations_reset_frequency
|
|
@@ -114,7 +109,7 @@ class Accumulation:
|
|
|
114
109
|
self.user_date = user_date
|
|
115
110
|
|
|
116
111
|
@property
|
|
117
|
-
def key(self) ->
|
|
112
|
+
def key(self) -> tuple[str, int, int, list[int], int]:
|
|
118
113
|
"""Returns the key for the accumulation."""
|
|
119
114
|
return (self.param, self.date, self.time, self.steps, self.number)
|
|
120
115
|
|
|
@@ -235,15 +230,15 @@ class Accumulation:
|
|
|
235
230
|
def mars_date_time_steps(
|
|
236
231
|
cls,
|
|
237
232
|
*,
|
|
238
|
-
dates:
|
|
233
|
+
dates: list[datetime.datetime],
|
|
239
234
|
step1: int,
|
|
240
235
|
step2: int,
|
|
241
|
-
frequency:
|
|
242
|
-
base_times:
|
|
236
|
+
frequency: int | None,
|
|
237
|
+
base_times: list[int],
|
|
243
238
|
adjust_step: bool,
|
|
244
|
-
accumulations_reset_frequency:
|
|
245
|
-
user_date:
|
|
246
|
-
) -> Generator[
|
|
239
|
+
accumulations_reset_frequency: int | None,
|
|
240
|
+
user_date: str | None,
|
|
241
|
+
) -> Generator[tuple[int, int, tuple[int, ...]], None, None]:
|
|
247
242
|
"""Generates MARS date-time steps.
|
|
248
243
|
|
|
249
244
|
Parameters
|
|
@@ -327,11 +322,11 @@ class Accumulation:
|
|
|
327
322
|
step1: int,
|
|
328
323
|
step2: int,
|
|
329
324
|
add_step: int,
|
|
330
|
-
frequency:
|
|
331
|
-
accumulations_reset_frequency:
|
|
332
|
-
user_date:
|
|
333
|
-
requested_date:
|
|
334
|
-
) ->
|
|
325
|
+
frequency: int | None,
|
|
326
|
+
accumulations_reset_frequency: int | None,
|
|
327
|
+
user_date: str | None,
|
|
328
|
+
requested_date: datetime.datetime | None = None,
|
|
329
|
+
) -> tuple[int, int, tuple[int, ...]]:
|
|
335
330
|
"""Generates a MARS date-time step.
|
|
336
331
|
|
|
337
332
|
Parameters
|
|
@@ -364,7 +359,7 @@ class Accumulation:
|
|
|
364
359
|
class AccumulationFromStart(Accumulation):
|
|
365
360
|
"""Class to handle data accumulation from the start of the forecast."""
|
|
366
361
|
|
|
367
|
-
def adjust_steps(self, startStep: int, endStep: int) ->
|
|
362
|
+
def adjust_steps(self, startStep: int, endStep: int) -> tuple[int, int]:
|
|
368
363
|
"""Adjusts the start and end steps.
|
|
369
364
|
|
|
370
365
|
Parameters
|
|
@@ -427,11 +422,11 @@ class AccumulationFromStart(Accumulation):
|
|
|
427
422
|
step1: int,
|
|
428
423
|
step2: int,
|
|
429
424
|
add_step: int,
|
|
430
|
-
frequency:
|
|
431
|
-
accumulations_reset_frequency:
|
|
432
|
-
user_date:
|
|
433
|
-
requested_date:
|
|
434
|
-
) ->
|
|
425
|
+
frequency: int | None,
|
|
426
|
+
accumulations_reset_frequency: int | None,
|
|
427
|
+
user_date: str | None,
|
|
428
|
+
requested_date: datetime.datetime | None = None,
|
|
429
|
+
) -> tuple[int, int, tuple[int, ...]]:
|
|
435
430
|
"""Generates a MARS date-time step.
|
|
436
431
|
|
|
437
432
|
Parameters
|
|
@@ -518,10 +513,10 @@ class AccumulationFromLastStep(Accumulation):
|
|
|
518
513
|
step2: int,
|
|
519
514
|
add_step: int,
|
|
520
515
|
frequency: int,
|
|
521
|
-
accumulations_reset_frequency:
|
|
522
|
-
user_date:
|
|
523
|
-
requested_date:
|
|
524
|
-
) ->
|
|
516
|
+
accumulations_reset_frequency: int | None,
|
|
517
|
+
user_date: str | None = None,
|
|
518
|
+
requested_date: datetime.datetime | None = None,
|
|
519
|
+
) -> tuple[int, int, tuple[int, ...]]:
|
|
525
520
|
"""Generates a MARS date-time step.
|
|
526
521
|
|
|
527
522
|
Parameters
|
|
@@ -568,7 +563,7 @@ class AccumulationFromLastStep(Accumulation):
|
|
|
568
563
|
class AccumulationFromLastReset(Accumulation):
|
|
569
564
|
"""Class to handle data accumulation from the last step of the forecast."""
|
|
570
565
|
|
|
571
|
-
def adjust_steps(self, startStep: int, endStep: int) ->
|
|
566
|
+
def adjust_steps(self, startStep: int, endStep: int) -> tuple[int, int]:
|
|
572
567
|
"""Adjusts the start and end steps.
|
|
573
568
|
|
|
574
569
|
Parameters
|
|
@@ -588,7 +583,7 @@ class AccumulationFromLastReset(Accumulation):
|
|
|
588
583
|
@classmethod
|
|
589
584
|
def _adjust_steps(
|
|
590
585
|
self, startStep: int, endStep: int, frequency: int, accumulations_reset_frequency: int
|
|
591
|
-
) ->
|
|
586
|
+
) -> tuple[int, int]:
|
|
592
587
|
"""Adjusts the start and end steps.
|
|
593
588
|
|
|
594
589
|
Parameters
|
|
@@ -620,7 +615,7 @@ class AccumulationFromLastReset(Accumulation):
|
|
|
620
615
|
base_date: datetime.datetime,
|
|
621
616
|
frequency: int,
|
|
622
617
|
accumulations_reset_frequency: int,
|
|
623
|
-
) ->
|
|
618
|
+
) -> tuple[int, int]:
|
|
624
619
|
"""Calculates the steps for accumulation.
|
|
625
620
|
|
|
626
621
|
Parameters
|
|
@@ -704,10 +699,10 @@ class AccumulationFromLastReset(Accumulation):
|
|
|
704
699
|
step2: int,
|
|
705
700
|
add_step: int,
|
|
706
701
|
frequency: int,
|
|
707
|
-
accumulations_reset_frequency:
|
|
708
|
-
user_date:
|
|
709
|
-
requested_date:
|
|
710
|
-
) ->
|
|
702
|
+
accumulations_reset_frequency: int | None,
|
|
703
|
+
user_date: str | None,
|
|
704
|
+
requested_date: datetime.datetime | None = None,
|
|
705
|
+
) -> tuple[int, int, tuple[int, ...]]:
|
|
711
706
|
"""Generates a MARS date-time step.
|
|
712
707
|
|
|
713
708
|
Parameters
|
|
@@ -776,15 +771,15 @@ def _identity(x: Any) -> Any:
|
|
|
776
771
|
|
|
777
772
|
def _compute_accumulations(
|
|
778
773
|
context: Any,
|
|
779
|
-
dates:
|
|
780
|
-
request:
|
|
781
|
-
user_accumulation_period:
|
|
782
|
-
data_accumulation_period:
|
|
783
|
-
accumulations_reset_frequency:
|
|
784
|
-
user_date:
|
|
774
|
+
dates: list[datetime.datetime],
|
|
775
|
+
request: dict[str, Any],
|
|
776
|
+
user_accumulation_period: int | tuple[int, int] = 6,
|
|
777
|
+
data_accumulation_period: int | None = None,
|
|
778
|
+
accumulations_reset_frequency: int | None = None,
|
|
779
|
+
user_date: str | None = None,
|
|
785
780
|
patch: Any = _identity,
|
|
786
|
-
base_times:
|
|
787
|
-
use_cdsapi_dataset:
|
|
781
|
+
base_times: list[int] | None = None,
|
|
782
|
+
use_cdsapi_dataset: str | None = None,
|
|
788
783
|
) -> Any:
|
|
789
784
|
"""Computes accumulations based on the provided parameters.
|
|
790
785
|
|
|
@@ -933,7 +928,7 @@ def _compute_accumulations(
|
|
|
933
928
|
return ds
|
|
934
929
|
|
|
935
930
|
|
|
936
|
-
def _to_list(x:
|
|
931
|
+
def _to_list(x: list[Any] | tuple[Any] | Any) -> list[Any]:
|
|
937
932
|
"""Converts the input to a list if it is not already a list or tuple.
|
|
938
933
|
|
|
939
934
|
Parameters
|
|
@@ -951,7 +946,7 @@ def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
|
|
|
951
946
|
return [x]
|
|
952
947
|
|
|
953
948
|
|
|
954
|
-
def _scda(request:
|
|
949
|
+
def _scda(request: dict[str, Any]) -> dict[str, Any]:
|
|
955
950
|
"""Modifies the request stream based on the time.
|
|
956
951
|
|
|
957
952
|
Parameters
|
|
@@ -973,7 +968,7 @@ def _scda(request: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
973
968
|
|
|
974
969
|
@legacy_source(__file__)
|
|
975
970
|
def accumulations(
|
|
976
|
-
context: Any, dates:
|
|
971
|
+
context: Any, dates: list[datetime.datetime], use_cdsapi_dataset: str | None = None, **request: Any
|
|
977
972
|
) -> Any:
|
|
978
973
|
"""Computes accumulations based on the provided context, dates, and request parameters.
|
|
979
974
|
|
|
@@ -12,10 +12,6 @@ import logging
|
|
|
12
12
|
from abc import abstractmethod
|
|
13
13
|
from copy import deepcopy
|
|
14
14
|
from typing import Any
|
|
15
|
-
from typing import Dict
|
|
16
|
-
from typing import List
|
|
17
|
-
from typing import Tuple
|
|
18
|
-
from typing import Union
|
|
19
15
|
|
|
20
16
|
import earthkit.data as ekd
|
|
21
17
|
import numpy as np
|
|
@@ -477,8 +473,8 @@ class Accumulator:
|
|
|
477
473
|
|
|
478
474
|
def _compute_accumulations(
|
|
479
475
|
context: Any,
|
|
480
|
-
dates:
|
|
481
|
-
request:
|
|
476
|
+
dates: list[datetime.datetime],
|
|
477
|
+
request: dict[str, Any],
|
|
482
478
|
user_accumulation_period: datetime.timedelta,
|
|
483
479
|
# data_accumulation_period: Optional[int] = None,
|
|
484
480
|
# patch: Any = _identity,
|
|
@@ -565,7 +561,7 @@ def _compute_accumulations(
|
|
|
565
561
|
return ds
|
|
566
562
|
|
|
567
563
|
|
|
568
|
-
def _to_list(x:
|
|
564
|
+
def _to_list(x: list[Any] | tuple[Any] | Any) -> list[Any]:
|
|
569
565
|
"""Converts the input to a list if it is not already a list or tuple.
|
|
570
566
|
|
|
571
567
|
Parameters
|
|
@@ -583,7 +579,7 @@ def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
|
|
|
583
579
|
return [x]
|
|
584
580
|
|
|
585
581
|
|
|
586
|
-
def _scda(request:
|
|
582
|
+
def _scda(request: dict[str, Any]) -> dict[str, Any]:
|
|
587
583
|
"""Modifies the request stream based on the time.
|
|
588
584
|
|
|
589
585
|
Parameters
|
|
@@ -8,8 +8,6 @@
|
|
|
8
8
|
# nor does it submit to any jurisdiction.
|
|
9
9
|
|
|
10
10
|
from typing import Any
|
|
11
|
-
from typing import Dict
|
|
12
|
-
from typing import List
|
|
13
11
|
|
|
14
12
|
from earthkit.data import from_source
|
|
15
13
|
|
|
@@ -17,7 +15,7 @@ from .legacy import legacy_source
|
|
|
17
15
|
|
|
18
16
|
|
|
19
17
|
@legacy_source(__file__)
|
|
20
|
-
def constants(context: Any, dates:
|
|
18
|
+
def constants(context: Any, dates: list[str], template: dict[str, Any], param: str) -> Any:
|
|
21
19
|
"""Deprecated function to retrieve constants data.
|
|
22
20
|
|
|
23
21
|
Parameters
|