anemoi-datasets 0.5.25__py3-none-any.whl → 0.5.27__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. anemoi/datasets/__init__.py +1 -2
  2. anemoi/datasets/_version.py +16 -3
  3. anemoi/datasets/commands/check.py +1 -1
  4. anemoi/datasets/commands/copy.py +1 -2
  5. anemoi/datasets/commands/create.py +1 -1
  6. anemoi/datasets/commands/grib-index.py +1 -1
  7. anemoi/datasets/commands/inspect.py +27 -35
  8. anemoi/datasets/commands/validate.py +59 -0
  9. anemoi/datasets/compute/recentre.py +3 -6
  10. anemoi/datasets/create/__init__.py +22 -25
  11. anemoi/datasets/create/check.py +10 -12
  12. anemoi/datasets/create/chunks.py +1 -2
  13. anemoi/datasets/create/config.py +3 -6
  14. anemoi/datasets/create/filter.py +21 -24
  15. anemoi/datasets/create/input/__init__.py +1 -2
  16. anemoi/datasets/create/input/action.py +3 -5
  17. anemoi/datasets/create/input/concat.py +5 -8
  18. anemoi/datasets/create/input/context.py +3 -6
  19. anemoi/datasets/create/input/data_sources.py +5 -8
  20. anemoi/datasets/create/input/empty.py +1 -2
  21. anemoi/datasets/create/input/filter.py +2 -3
  22. anemoi/datasets/create/input/function.py +1 -2
  23. anemoi/datasets/create/input/join.py +4 -5
  24. anemoi/datasets/create/input/misc.py +4 -6
  25. anemoi/datasets/create/input/repeated_dates.py +13 -18
  26. anemoi/datasets/create/input/result.py +29 -33
  27. anemoi/datasets/create/input/step.py +6 -24
  28. anemoi/datasets/create/input/template.py +3 -4
  29. anemoi/datasets/create/input/trace.py +1 -1
  30. anemoi/datasets/create/patch.py +1 -2
  31. anemoi/datasets/create/persistent.py +3 -5
  32. anemoi/datasets/create/size.py +1 -3
  33. anemoi/datasets/create/sources/accumulations.py +47 -52
  34. anemoi/datasets/create/sources/accumulations2.py +4 -8
  35. anemoi/datasets/create/sources/constants.py +1 -3
  36. anemoi/datasets/create/sources/empty.py +1 -2
  37. anemoi/datasets/create/sources/fdb.py +133 -0
  38. anemoi/datasets/create/sources/forcings.py +1 -2
  39. anemoi/datasets/create/sources/grib.py +6 -10
  40. anemoi/datasets/create/sources/grib_index.py +13 -15
  41. anemoi/datasets/create/sources/hindcasts.py +2 -5
  42. anemoi/datasets/create/sources/legacy.py +1 -1
  43. anemoi/datasets/create/sources/mars.py +17 -21
  44. anemoi/datasets/create/sources/netcdf.py +1 -2
  45. anemoi/datasets/create/sources/opendap.py +1 -3
  46. anemoi/datasets/create/sources/patterns.py +4 -6
  47. anemoi/datasets/create/sources/planetary_computer.py +44 -0
  48. anemoi/datasets/create/sources/recentre.py +8 -11
  49. anemoi/datasets/create/sources/source.py +3 -6
  50. anemoi/datasets/create/sources/tendencies.py +2 -5
  51. anemoi/datasets/create/sources/xarray.py +4 -6
  52. anemoi/datasets/create/sources/xarray_support/__init__.py +15 -32
  53. anemoi/datasets/create/sources/xarray_support/coordinates.py +16 -12
  54. anemoi/datasets/create/sources/xarray_support/field.py +17 -16
  55. anemoi/datasets/create/sources/xarray_support/fieldlist.py +11 -15
  56. anemoi/datasets/create/sources/xarray_support/flavour.py +83 -45
  57. anemoi/datasets/create/sources/xarray_support/grid.py +15 -9
  58. anemoi/datasets/create/sources/xarray_support/metadata.py +19 -128
  59. anemoi/datasets/create/sources/xarray_support/patch.py +47 -6
  60. anemoi/datasets/create/sources/xarray_support/time.py +10 -13
  61. anemoi/datasets/create/sources/xarray_support/variable.py +27 -23
  62. anemoi/datasets/create/sources/xarray_zarr.py +1 -2
  63. anemoi/datasets/create/sources/zenodo.py +3 -5
  64. anemoi/datasets/create/statistics/__init__.py +3 -6
  65. anemoi/datasets/create/testing.py +2 -74
  66. anemoi/datasets/create/typing.py +1 -2
  67. anemoi/datasets/create/utils.py +1 -2
  68. anemoi/datasets/create/zarr.py +7 -2
  69. anemoi/datasets/data/__init__.py +15 -6
  70. anemoi/datasets/data/complement.py +52 -23
  71. anemoi/datasets/data/concat.py +5 -8
  72. anemoi/datasets/data/dataset.py +42 -47
  73. anemoi/datasets/data/debug.py +7 -9
  74. anemoi/datasets/data/ensemble.py +4 -6
  75. anemoi/datasets/data/fill_missing.py +7 -10
  76. anemoi/datasets/data/forwards.py +30 -28
  77. anemoi/datasets/data/grids.py +12 -16
  78. anemoi/datasets/data/indexing.py +9 -12
  79. anemoi/datasets/data/interpolate.py +7 -15
  80. anemoi/datasets/data/join.py +8 -12
  81. anemoi/datasets/data/masked.py +6 -11
  82. anemoi/datasets/data/merge.py +5 -9
  83. anemoi/datasets/data/misc.py +41 -45
  84. anemoi/datasets/data/missing.py +11 -16
  85. anemoi/datasets/data/observations/__init__.py +8 -14
  86. anemoi/datasets/data/padded.py +3 -5
  87. anemoi/datasets/data/records/backends/__init__.py +2 -2
  88. anemoi/datasets/data/rescale.py +5 -12
  89. anemoi/datasets/data/select.py +13 -16
  90. anemoi/datasets/data/statistics.py +4 -7
  91. anemoi/datasets/data/stores.py +23 -77
  92. anemoi/datasets/data/subset.py +8 -11
  93. anemoi/datasets/data/unchecked.py +7 -11
  94. anemoi/datasets/data/xy.py +25 -21
  95. anemoi/datasets/dates/__init__.py +13 -18
  96. anemoi/datasets/dates/groups.py +7 -10
  97. anemoi/datasets/grids.py +11 -12
  98. anemoi/datasets/testing.py +93 -7
  99. anemoi/datasets/validate.py +598 -0
  100. {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/METADATA +5 -4
  101. anemoi_datasets-0.5.27.dist-info/RECORD +134 -0
  102. anemoi/datasets/create/filters/__init__.py +0 -33
  103. anemoi/datasets/create/filters/empty.py +0 -37
  104. anemoi/datasets/create/filters/legacy.py +0 -93
  105. anemoi/datasets/create/filters/noop.py +0 -37
  106. anemoi/datasets/create/filters/orog_to_z.py +0 -58
  107. anemoi/datasets/create/filters/pressure_level_relative_humidity_to_specific_humidity.py +0 -83
  108. anemoi/datasets/create/filters/pressure_level_specific_humidity_to_relative_humidity.py +0 -84
  109. anemoi/datasets/create/filters/rename.py +0 -205
  110. anemoi/datasets/create/filters/rotate_winds.py +0 -105
  111. anemoi/datasets/create/filters/single_level_dewpoint_to_relative_humidity.py +0 -78
  112. anemoi/datasets/create/filters/single_level_relative_humidity_to_dewpoint.py +0 -84
  113. anemoi/datasets/create/filters/single_level_relative_humidity_to_specific_humidity.py +0 -163
  114. anemoi/datasets/create/filters/single_level_specific_humidity_to_relative_humidity.py +0 -451
  115. anemoi/datasets/create/filters/speeddir_to_uv.py +0 -95
  116. anemoi/datasets/create/filters/sum.py +0 -68
  117. anemoi/datasets/create/filters/transform.py +0 -51
  118. anemoi/datasets/create/filters/unrotate_winds.py +0 -105
  119. anemoi/datasets/create/filters/uv_to_speeddir.py +0 -94
  120. anemoi/datasets/create/filters/wz_to_w.py +0 -98
  121. anemoi/datasets/utils/__init__.py +0 -8
  122. anemoi_datasets-0.5.25.dist-info/RECORD +0 -150
  123. {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/WHEEL +0 -0
  124. {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/entry_points.txt +0 -0
  125. {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/licenses/LICENSE +0 -0
  126. {anemoi_datasets-0.5.25.dist-info → anemoi_datasets-0.5.27.dist-info}/top_level.txt +0 -0
@@ -15,10 +15,6 @@ from collections import defaultdict
15
15
  from functools import cached_property
16
16
  from typing import Any
17
17
  from typing import DefaultDict
18
- from typing import Dict
19
- from typing import List
20
- from typing import Optional
21
- from typing import Tuple
22
18
 
23
19
  import numpy as np
24
20
  from anemoi.utils.dates import as_timedelta
@@ -33,7 +29,7 @@ from .trace import trace_datasource
33
29
  LOG = logging.getLogger(__name__)
34
30
 
35
31
 
36
- def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
32
+ def _fields_metatata(variables: tuple[str, ...], cube: Any) -> dict[str, Any]:
37
33
  """Retrieve metadata for the given variables and cube.
38
34
 
39
35
  Parameters
@@ -50,7 +46,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
50
46
  """
51
47
  assert isinstance(variables, tuple), variables
52
48
 
53
- KNOWN: Dict[str, Dict[str, bool]] = {
49
+ KNOWN: dict[str, dict[str, bool]] = {
54
50
  "cos_julian_day": dict(computed_forcing=True, constant_in_time=False),
55
51
  "cos_latitude": dict(computed_forcing=True, constant_in_time=True),
56
52
  "cos_local_time": dict(computed_forcing=True, constant_in_time=False),
@@ -65,9 +61,9 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
65
61
  "sin_longitude": dict(computed_forcing=True, constant_in_time=True),
66
62
  }
67
63
 
68
- def _merge(md1: Dict[str, Any], md2: Dict[str, Any]) -> Dict[str, Any]:
64
+ def _merge(md1: dict[str, Any], md2: dict[str, Any]) -> dict[str, Any]:
69
65
  assert set(md1.keys()) == set(md2.keys()), (set(md1.keys()), set(md2.keys()))
70
- result: Dict[str, Any] = {}
66
+ result: dict[str, Any] = {}
71
67
  for k in md1.keys():
72
68
  v1 = md1[k]
73
69
  v2 = md2[k]
@@ -90,10 +86,10 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
90
86
 
91
87
  return result
92
88
 
93
- mars: Dict[str, Any] = {}
94
- other: DefaultDict[str, Dict[str, Any]] = defaultdict(dict)
89
+ mars: dict[str, Any] = {}
90
+ other: DefaultDict[str, dict[str, Any]] = defaultdict(dict)
95
91
  i: int = -1
96
- date: Optional[str] = None
92
+ date: str | None = None
97
93
  for c in cube.iterate_cubelets():
98
94
 
99
95
  if date is None:
@@ -137,7 +133,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
137
133
 
138
134
  if startStep != endStep:
139
135
  # https://codes.ecmwf.int/grib/format/grib2/ctables/4/10/
140
- TYPE_OF_STATISTICAL_PROCESSING: Dict[Optional[int], Optional[str]] = {
136
+ TYPE_OF_STATISTICAL_PROCESSING: dict[int | None, str | None] = {
141
137
  None: None,
142
138
  0: "average",
143
139
  1: "accumulation",
@@ -157,12 +153,12 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
157
153
 
158
154
  # https://codes.ecmwf.int/grib/format/grib1/ctable/5/
159
155
 
160
- TIME_RANGE_INDICATOR: Dict[int, str] = {
156
+ TIME_RANGE_INDICATOR: dict[int, str] = {
161
157
  4: "accumulation",
162
158
  3: "average",
163
159
  }
164
160
 
165
- STEP_TYPE_FOR_CONVERSION: Dict[str, str] = {
161
+ STEP_TYPE_FOR_CONVERSION: dict[str, str] = {
166
162
  "min": "minimum",
167
163
  "max": "maximum",
168
164
  "accum": "accumulation",
@@ -172,7 +168,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
172
168
  # A few patches
173
169
  #
174
170
 
175
- PATCHES: Dict[str, str] = {
171
+ PATCHES: dict[str, str] = {
176
172
  "10fg6": "maximum",
177
173
  "mntpr3": "minimum", # Not in param db
178
174
  "mntpr6": "minimum", # Not in param db
@@ -211,7 +207,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
211
207
  else:
212
208
  mars[variables[i]] = md
213
209
 
214
- result: Dict[str, Dict[str, Any]] = {}
210
+ result: dict[str, dict[str, Any]] = {}
215
211
  for k, v in mars.items():
216
212
  result[k] = dict(mars=v) if v else {}
217
213
  result[k].update(other[k])
@@ -222,7 +218,7 @@ def _fields_metatata(variables: Tuple[str, ...], cube: Any) -> Dict[str, Any]:
222
218
  return result
223
219
 
224
220
 
225
- def _data_request(data: Any) -> Dict[str, Any]:
221
+ def _data_request(data: Any) -> dict[str, Any]:
226
222
  """Build a data request dictionary from the given data.
227
223
 
228
224
  Parameters
@@ -235,12 +231,12 @@ def _data_request(data: Any) -> Dict[str, Any]:
235
231
  dict
236
232
  The data request dictionary.
237
233
  """
238
- date: Optional[Any] = None
234
+ date: Any | None = None
239
235
  params_levels: DefaultDict[str, set] = defaultdict(set)
240
236
  params_steps: DefaultDict[str, set] = defaultdict(set)
241
237
 
242
- area: Optional[Any] = None
243
- grid: Optional[Any] = None
238
+ area: Any | None = None
239
+ grid: Any | None = None
244
240
 
245
241
  for field in data:
246
242
  try:
@@ -270,8 +266,8 @@ def _data_request(data: Any) -> Dict[str, Any]:
270
266
  except Exception:
271
267
  LOG.error(f"Error in retrieving metadata (cannot build data request info) for {field}", exc_info=True)
272
268
 
273
- def sort(old_dic: DefaultDict[str, set]) -> Dict[str, List[Any]]:
274
- new_dic: Dict[str, List[Any]] = {}
269
+ def sort(old_dic: DefaultDict[str, set]) -> dict[str, list[Any]]:
270
+ new_dic: dict[str, list[Any]] = {}
275
271
  for k, v in old_dic.items():
276
272
  new_dic[k] = sorted(list(v))
277
273
  return new_dic
@@ -288,7 +284,7 @@ class Result:
288
284
  empty: bool = False
289
285
  _coords_already_built: bool = False
290
286
 
291
- def __init__(self, context: ActionContext, action_path: List[str], dates: Any) -> None:
287
+ def __init__(self, context: ActionContext, action_path: list[str], dates: Any) -> None:
292
288
  """Initialize a Result instance.
293
289
 
294
290
  Parameters
@@ -309,7 +305,7 @@ class Result:
309
305
 
310
306
  self.context: Any = context
311
307
  self.group_of_dates: Any = dates
312
- self.action_path: List[str] = action_path
308
+ self.action_path: list[str] = action_path
313
309
 
314
310
  @property
315
311
  @trace_datasource
@@ -318,7 +314,7 @@ class Result:
318
314
  self._raise_not_implemented()
319
315
 
320
316
  @property
321
- def data_request(self) -> Dict[str, Any]:
317
+ def data_request(self) -> dict[str, Any]:
322
318
  """Returns a dictionary with the parameters needed to retrieve the data."""
323
319
  return _data_request(self.datasource)
324
320
 
@@ -340,7 +336,7 @@ class Result:
340
336
  LOG.debug("Sorting dataset %s %s", dict(order_by), remapping)
341
337
  assert order_by, order_by
342
338
 
343
- patches: Dict[str, Dict[Optional[Any], int]] = {"number": {None: 0}}
339
+ patches: dict[str, dict[Any | None, int]] = {"number": {None: 0}}
344
340
 
345
341
  try:
346
342
  cube: Any = ds.cube(
@@ -377,7 +373,7 @@ class Result:
377
373
  patches : Any
378
374
  The patches configuration.
379
375
  """
380
- METADATA: Tuple[str, ...] = (
376
+ METADATA: tuple[str, ...] = (
381
377
  "date",
382
378
  "time",
383
379
  "step",
@@ -402,7 +398,7 @@ class Result:
402
398
  # print("Executing", self.action_path)
403
399
  # print("Dates:", compress_dates(self.dates))
404
400
 
405
- names: List[str] = []
401
+ names: list[str] = []
406
402
  for a in args:
407
403
  if isinstance(a, str):
408
404
  names.append(a)
@@ -421,7 +417,7 @@ class Result:
421
417
  print(" ", n)
422
418
 
423
419
  print()
424
- user_shape: Tuple[int, ...] = tuple(len(v) for k, v in user_coords.items())
420
+ user_shape: tuple[int, ...] = tuple(len(v) for k, v in user_coords.items())
425
421
  print("Shape of the hypercube :", user_shape)
426
422
  print(
427
423
  "Number of expected fields :", math.prod(user_shape), "=", " x ".join([str(i) for i in user_shape])
@@ -643,13 +639,13 @@ class Result:
643
639
  self._coords_already_built: bool = True
644
640
 
645
641
  @property
646
- def variables(self) -> List[str]:
642
+ def variables(self) -> list[str]:
647
643
  """Retrieve the variables for the result."""
648
644
  self.build_coords()
649
645
  return self._variables
650
646
 
651
647
  @property
652
- def variables_metadata(self) -> Dict[str, Any]:
648
+ def variables_metadata(self) -> dict[str, Any]:
653
649
  """Retrieve the metadata for the variables."""
654
650
  return _fields_metatata(self.variables, self._cube)
655
651
 
@@ -690,7 +686,7 @@ class Result:
690
686
  return self._proj_string
691
687
 
692
688
  @cached_property
693
- def shape(self) -> List[int]:
689
+ def shape(self) -> list[int]:
694
690
  """Retrieve the shape of the result."""
695
691
  return [
696
692
  len(self.group_of_dates),
@@ -700,7 +696,7 @@ class Result:
700
696
  ]
701
697
 
702
698
  @cached_property
703
- def coords(self) -> Dict[str, Any]:
699
+ def coords(self) -> dict[str, Any]:
704
700
  """Retrieve the coordinates of the result."""
705
701
  return {
706
702
  "dates": list(self.group_of_dates),
@@ -8,13 +8,8 @@
8
8
  # nor does it submit to any jurisdiction.
9
9
 
10
10
  import logging
11
- import warnings
12
11
  from copy import deepcopy
13
12
  from typing import Any
14
- from typing import Dict
15
- from typing import List
16
- from typing import Optional
17
- from typing import Type
18
13
 
19
14
  from .action import Action
20
15
  from .action import ActionContext
@@ -31,7 +26,7 @@ class StepResult(Result):
31
26
  """Represents the result of a step in the data processing pipeline."""
32
27
 
33
28
  def __init__(
34
- self, context: Context, action_path: List[str], group_of_dates: Any, action: Action, upstream_result: Result
29
+ self, context: Context, action_path: list[str], group_of_dates: Any, action: Action, upstream_result: Result
35
30
  ) -> None:
36
31
  """Initialize a StepResult instance.
37
32
 
@@ -64,10 +59,10 @@ class StepResult(Result):
64
59
  class StepAction(Action):
65
60
  """Represents an action that is part of a step in the data processing pipeline."""
66
61
 
67
- result_class: Optional[Type[StepResult]] = None
62
+ result_class: type[StepResult] | None = None
68
63
 
69
64
  def __init__(
70
- self, context: ActionContext, action_path: List[str], previous_step: Any, *args: Any, **kwargs: Any
65
+ self, context: ActionContext, action_path: list[str], previous_step: Any, *args: Any, **kwargs: Any
71
66
  ) -> None:
72
67
  """Initialize a StepAction instance.
73
68
 
@@ -116,7 +111,7 @@ class StepAction(Action):
116
111
  return self._repr(self.previous_step, _inline_=str(self.kwargs))
117
112
 
118
113
 
119
- def step_factory(config: Dict[str, Any], context: ActionContext, action_path: List[str], previous_step: Any) -> Any:
114
+ def step_factory(config: dict[str, Any], context: ActionContext, action_path: list[str], previous_step: Any) -> Any:
120
115
  """Factory function to create a step action based on the given configuration.
121
116
 
122
117
  Parameters
@@ -165,24 +160,11 @@ def step_factory(config: Dict[str, Any], context: ActionContext, action_path: Li
165
160
  if cls is not None:
166
161
  return cls(context, action_path, previous_step, *args, **kwargs)
167
162
 
168
- # Try filters from datasets filter registry
163
+ # Try filters from transform filter registry
169
164
  from anemoi.transform.filters import filter_registry as transform_filter_registry
170
165
 
171
- from ..filters import create_filter as create_datasets_filter
172
- from ..filters import filter_registry as datasets_filter_registry
173
-
174
- if datasets_filter_registry.is_registered(key):
175
-
176
- if transform_filter_registry.is_registered(key):
177
- warnings.warn(f"Filter `{key}` is registered in both datasets and transform filter registries")
178
-
179
- filter = create_datasets_filter(None, config)
180
- return FunctionStepAction(context, action_path + [key], previous_step, key, filter)
181
-
182
- # Use filters from transform registry
183
-
184
166
  if transform_filter_registry.is_registered(key):
185
- from ..filters.transform import TransformFilter
167
+ from ..filter import TransformFilter
186
168
 
187
169
  return FunctionStepAction(
188
170
  context, action_path + [key], previous_step, key, TransformFilter(context, key, config)
@@ -12,10 +12,9 @@ import logging
12
12
  import re
13
13
  from abc import ABC
14
14
  from abc import abstractmethod
15
+ from collections.abc import Callable
15
16
  from functools import wraps
16
17
  from typing import Any
17
- from typing import Callable
18
- from typing import List
19
18
 
20
19
  from .context import Context
21
20
 
@@ -68,7 +67,7 @@ class Substitution(ABC):
68
67
  class Reference(Substitution):
69
68
  """A class to represent a reference to another value in the context."""
70
69
 
71
- def __init__(self, context: Any, action_path: List[str]) -> None:
70
+ def __init__(self, context: Any, action_path: list[str]) -> None:
72
71
  """Initialize a Reference instance.
73
72
 
74
73
  Parameters
@@ -79,7 +78,7 @@ class Reference(Substitution):
79
78
  The action path to resolve.
80
79
  """
81
80
  self.context: Any = context
82
- self.action_path: List[str] = action_path
81
+ self.action_path: list[str] = action_path
83
82
 
84
83
  def resolve(self, context: Context) -> Any:
85
84
  """Resolve the reference using the given context.
@@ -11,9 +11,9 @@
11
11
  import logging
12
12
  import textwrap
13
13
  import threading
14
+ from collections.abc import Callable
14
15
  from functools import wraps
15
16
  from typing import Any
16
- from typing import Callable
17
17
 
18
18
  LOG = logging.getLogger(__name__)
19
19
 
@@ -10,14 +10,13 @@
10
10
  import json
11
11
  import logging
12
12
  import os
13
- from typing import Union
14
13
 
15
14
  import zarr
16
15
 
17
16
  LOG = logging.getLogger(__name__)
18
17
 
19
18
 
20
- def fix_order_by(order_by: Union[dict, list]) -> list[dict]:
19
+ def fix_order_by(order_by: dict | list) -> list[dict]:
21
20
  """Fix the order_by attribute to ensure it is a list of dictionaries.
22
21
 
23
22
  Parameters
@@ -16,9 +16,8 @@ import os
16
16
  import pickle
17
17
  import shutil
18
18
  import socket
19
+ from collections.abc import Iterator
19
20
  from typing import Any
20
- from typing import Iterator
21
- from typing import Tuple
22
21
 
23
22
  import numpy as np
24
23
  from anemoi.utils.provenance import gather_provenance_info
@@ -198,7 +197,7 @@ class BufferedPersistentDict(PersistentDict):
198
197
  self.elements = []
199
198
  self.keys = []
200
199
 
201
- def items(self) -> Iterator[Tuple[Any, Any]]:
200
+ def items(self) -> Iterator[tuple[Any, Any]]:
202
201
  """Yield items stored in the BufferedPersistentDict.
203
202
 
204
203
  Yields
@@ -207,8 +206,7 @@ class BufferedPersistentDict(PersistentDict):
207
206
  An iterator over the items.
208
207
  """
209
208
  for keys, elements in self.storage.items():
210
- for key, elt in zip(keys, elements):
211
- yield key, elt
209
+ yield from zip(keys, elements)
212
210
 
213
211
  def delete(self) -> None:
214
212
  """Delete the storage directory and its contents."""
@@ -10,8 +10,6 @@
10
10
 
11
11
  import logging
12
12
  import os
13
- from typing import Dict
14
- from typing import Optional
15
13
 
16
14
  import tqdm
17
15
  from anemoi.utils.humanize import bytes_to_human
@@ -19,7 +17,7 @@ from anemoi.utils.humanize import bytes_to_human
19
17
  LOG = logging.getLogger(__name__)
20
18
 
21
19
 
22
- def compute_directory_sizes(path: str) -> Optional[Dict[str, int]]:
20
+ def compute_directory_sizes(path: str) -> dict[str, int] | None:
23
21
  """Computes the total size and number of files in a directory.
24
22
 
25
23
  Parameters
@@ -10,14 +10,9 @@
10
10
  import datetime
11
11
  import logging
12
12
  import warnings
13
+ from collections.abc import Generator
13
14
  from copy import deepcopy
14
15
  from typing import Any
15
- from typing import Dict
16
- from typing import Generator
17
- from typing import List
18
- from typing import Optional
19
- from typing import Tuple
20
- from typing import Union
21
16
 
22
17
  import earthkit.data as ekd
23
18
  import numpy as np
@@ -66,10 +61,10 @@ class Accumulation:
66
61
  date: int,
67
62
  time: int,
68
63
  number: int,
69
- step: List[int],
64
+ step: list[int],
70
65
  frequency: int,
71
- accumulations_reset_frequency: Optional[int] = None,
72
- user_date: Optional[str] = None,
66
+ accumulations_reset_frequency: int | None = None,
67
+ user_date: str | None = None,
73
68
  **kwargs: Any,
74
69
  ) -> None:
75
70
  """Initialises an Accumulation instance.
@@ -103,10 +98,10 @@ class Accumulation:
103
98
  self.time = time
104
99
  self.steps = step
105
100
  self.number = number
106
- self.values: Optional[NDArray[None]] = None
101
+ self.values: NDArray[None] | None = None
107
102
  self.seen = set()
108
- self.startStep: Optional[int] = None
109
- self.endStep: Optional[int] = None
103
+ self.startStep: int | None = None
104
+ self.endStep: int | None = None
110
105
  self.done = False
111
106
  self.frequency = frequency
112
107
  self.accumulations_reset_frequency = accumulations_reset_frequency
@@ -114,7 +109,7 @@ class Accumulation:
114
109
  self.user_date = user_date
115
110
 
116
111
  @property
117
- def key(self) -> Tuple[str, int, int, List[int], int]:
112
+ def key(self) -> tuple[str, int, int, list[int], int]:
118
113
  """Returns the key for the accumulation."""
119
114
  return (self.param, self.date, self.time, self.steps, self.number)
120
115
 
@@ -235,15 +230,15 @@ class Accumulation:
235
230
  def mars_date_time_steps(
236
231
  cls,
237
232
  *,
238
- dates: List[datetime.datetime],
233
+ dates: list[datetime.datetime],
239
234
  step1: int,
240
235
  step2: int,
241
- frequency: Optional[int],
242
- base_times: List[int],
236
+ frequency: int | None,
237
+ base_times: list[int],
243
238
  adjust_step: bool,
244
- accumulations_reset_frequency: Optional[int],
245
- user_date: Optional[str],
246
- ) -> Generator[Tuple[int, int, Tuple[int, ...]], None, None]:
239
+ accumulations_reset_frequency: int | None,
240
+ user_date: str | None,
241
+ ) -> Generator[tuple[int, int, tuple[int, ...]], None, None]:
247
242
  """Generates MARS date-time steps.
248
243
 
249
244
  Parameters
@@ -327,11 +322,11 @@ class Accumulation:
327
322
  step1: int,
328
323
  step2: int,
329
324
  add_step: int,
330
- frequency: Optional[int],
331
- accumulations_reset_frequency: Optional[int],
332
- user_date: Optional[str],
333
- requested_date: Optional[datetime.datetime] = None,
334
- ) -> Tuple[int, int, Tuple[int, ...]]:
325
+ frequency: int | None,
326
+ accumulations_reset_frequency: int | None,
327
+ user_date: str | None,
328
+ requested_date: datetime.datetime | None = None,
329
+ ) -> tuple[int, int, tuple[int, ...]]:
335
330
  """Generates a MARS date-time step.
336
331
 
337
332
  Parameters
@@ -364,7 +359,7 @@ class Accumulation:
364
359
  class AccumulationFromStart(Accumulation):
365
360
  """Class to handle data accumulation from the start of the forecast."""
366
361
 
367
- def adjust_steps(self, startStep: int, endStep: int) -> Tuple[int, int]:
362
+ def adjust_steps(self, startStep: int, endStep: int) -> tuple[int, int]:
368
363
  """Adjusts the start and end steps.
369
364
 
370
365
  Parameters
@@ -427,11 +422,11 @@ class AccumulationFromStart(Accumulation):
427
422
  step1: int,
428
423
  step2: int,
429
424
  add_step: int,
430
- frequency: Optional[int],
431
- accumulations_reset_frequency: Optional[int],
432
- user_date: Optional[str],
433
- requested_date: Optional[datetime.datetime] = None,
434
- ) -> Tuple[int, int, Tuple[int, ...]]:
425
+ frequency: int | None,
426
+ accumulations_reset_frequency: int | None,
427
+ user_date: str | None,
428
+ requested_date: datetime.datetime | None = None,
429
+ ) -> tuple[int, int, tuple[int, ...]]:
435
430
  """Generates a MARS date-time step.
436
431
 
437
432
  Parameters
@@ -518,10 +513,10 @@ class AccumulationFromLastStep(Accumulation):
518
513
  step2: int,
519
514
  add_step: int,
520
515
  frequency: int,
521
- accumulations_reset_frequency: Optional[int],
522
- user_date: Optional[str] = None,
523
- requested_date: Optional[datetime.datetime] = None,
524
- ) -> Tuple[int, int, Tuple[int, ...]]:
516
+ accumulations_reset_frequency: int | None,
517
+ user_date: str | None = None,
518
+ requested_date: datetime.datetime | None = None,
519
+ ) -> tuple[int, int, tuple[int, ...]]:
525
520
  """Generates a MARS date-time step.
526
521
 
527
522
  Parameters
@@ -568,7 +563,7 @@ class AccumulationFromLastStep(Accumulation):
568
563
  class AccumulationFromLastReset(Accumulation):
569
564
  """Class to handle data accumulation from the last step of the forecast."""
570
565
 
571
- def adjust_steps(self, startStep: int, endStep: int) -> Tuple[int, int]:
566
+ def adjust_steps(self, startStep: int, endStep: int) -> tuple[int, int]:
572
567
  """Adjusts the start and end steps.
573
568
 
574
569
  Parameters
@@ -588,7 +583,7 @@ class AccumulationFromLastReset(Accumulation):
588
583
  @classmethod
589
584
  def _adjust_steps(
590
585
  self, startStep: int, endStep: int, frequency: int, accumulations_reset_frequency: int
591
- ) -> Tuple[int, int]:
586
+ ) -> tuple[int, int]:
592
587
  """Adjusts the start and end steps.
593
588
 
594
589
  Parameters
@@ -620,7 +615,7 @@ class AccumulationFromLastReset(Accumulation):
620
615
  base_date: datetime.datetime,
621
616
  frequency: int,
622
617
  accumulations_reset_frequency: int,
623
- ) -> Tuple[int, int]:
618
+ ) -> tuple[int, int]:
624
619
  """Calculates the steps for accumulation.
625
620
 
626
621
  Parameters
@@ -704,10 +699,10 @@ class AccumulationFromLastReset(Accumulation):
704
699
  step2: int,
705
700
  add_step: int,
706
701
  frequency: int,
707
- accumulations_reset_frequency: Optional[int],
708
- user_date: Optional[str],
709
- requested_date: Optional[datetime.datetime] = None,
710
- ) -> Tuple[int, int, Tuple[int, ...]]:
702
+ accumulations_reset_frequency: int | None,
703
+ user_date: str | None,
704
+ requested_date: datetime.datetime | None = None,
705
+ ) -> tuple[int, int, tuple[int, ...]]:
711
706
  """Generates a MARS date-time step.
712
707
 
713
708
  Parameters
@@ -776,15 +771,15 @@ def _identity(x: Any) -> Any:
776
771
 
777
772
  def _compute_accumulations(
778
773
  context: Any,
779
- dates: List[datetime.datetime],
780
- request: Dict[str, Any],
781
- user_accumulation_period: Union[int, Tuple[int, int]] = 6,
782
- data_accumulation_period: Optional[int] = None,
783
- accumulations_reset_frequency: Optional[int] = None,
784
- user_date: Optional[str] = None,
774
+ dates: list[datetime.datetime],
775
+ request: dict[str, Any],
776
+ user_accumulation_period: int | tuple[int, int] = 6,
777
+ data_accumulation_period: int | None = None,
778
+ accumulations_reset_frequency: int | None = None,
779
+ user_date: str | None = None,
785
780
  patch: Any = _identity,
786
- base_times: Optional[List[int]] = None,
787
- use_cdsapi_dataset: Optional[str] = None,
781
+ base_times: list[int] | None = None,
782
+ use_cdsapi_dataset: str | None = None,
788
783
  ) -> Any:
789
784
  """Computes accumulations based on the provided parameters.
790
785
 
@@ -933,7 +928,7 @@ def _compute_accumulations(
933
928
  return ds
934
929
 
935
930
 
936
- def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
931
+ def _to_list(x: list[Any] | tuple[Any] | Any) -> list[Any]:
937
932
  """Converts the input to a list if it is not already a list or tuple.
938
933
 
939
934
  Parameters
@@ -951,7 +946,7 @@ def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
951
946
  return [x]
952
947
 
953
948
 
954
- def _scda(request: Dict[str, Any]) -> Dict[str, Any]:
949
+ def _scda(request: dict[str, Any]) -> dict[str, Any]:
955
950
  """Modifies the request stream based on the time.
956
951
 
957
952
  Parameters
@@ -973,7 +968,7 @@ def _scda(request: Dict[str, Any]) -> Dict[str, Any]:
973
968
 
974
969
  @legacy_source(__file__)
975
970
  def accumulations(
976
- context: Any, dates: List[datetime.datetime], use_cdsapi_dataset: Optional[str] = None, **request: Any
971
+ context: Any, dates: list[datetime.datetime], use_cdsapi_dataset: str | None = None, **request: Any
977
972
  ) -> Any:
978
973
  """Computes accumulations based on the provided context, dates, and request parameters.
979
974
 
@@ -12,10 +12,6 @@ import logging
12
12
  from abc import abstractmethod
13
13
  from copy import deepcopy
14
14
  from typing import Any
15
- from typing import Dict
16
- from typing import List
17
- from typing import Tuple
18
- from typing import Union
19
15
 
20
16
  import earthkit.data as ekd
21
17
  import numpy as np
@@ -477,8 +473,8 @@ class Accumulator:
477
473
 
478
474
  def _compute_accumulations(
479
475
  context: Any,
480
- dates: List[datetime.datetime],
481
- request: Dict[str, Any],
476
+ dates: list[datetime.datetime],
477
+ request: dict[str, Any],
482
478
  user_accumulation_period: datetime.timedelta,
483
479
  # data_accumulation_period: Optional[int] = None,
484
480
  # patch: Any = _identity,
@@ -565,7 +561,7 @@ def _compute_accumulations(
565
561
  return ds
566
562
 
567
563
 
568
- def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
564
+ def _to_list(x: list[Any] | tuple[Any] | Any) -> list[Any]:
569
565
  """Converts the input to a list if it is not already a list or tuple.
570
566
 
571
567
  Parameters
@@ -583,7 +579,7 @@ def _to_list(x: Union[List[Any], Tuple[Any], Any]) -> List[Any]:
583
579
  return [x]
584
580
 
585
581
 
586
- def _scda(request: Dict[str, Any]) -> Dict[str, Any]:
582
+ def _scda(request: dict[str, Any]) -> dict[str, Any]:
587
583
  """Modifies the request stream based on the time.
588
584
 
589
585
  Parameters
@@ -8,8 +8,6 @@
8
8
  # nor does it submit to any jurisdiction.
9
9
 
10
10
  from typing import Any
11
- from typing import Dict
12
- from typing import List
13
11
 
14
12
  from earthkit.data import from_source
15
13
 
@@ -17,7 +15,7 @@ from .legacy import legacy_source
17
15
 
18
16
 
19
17
  @legacy_source(__file__)
20
- def constants(context: Any, dates: List[str], template: Dict[str, Any], param: str) -> Any:
18
+ def constants(context: Any, dates: list[str], template: dict[str, Any], param: str) -> Any:
21
19
  """Deprecated function to retrieve constants data.
22
20
 
23
21
  Parameters