anemoi-datasets 0.5.16__py3-none-any.whl → 0.5.17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. anemoi/datasets/__init__.py +4 -1
  2. anemoi/datasets/__main__.py +12 -2
  3. anemoi/datasets/_version.py +9 -4
  4. anemoi/datasets/commands/cleanup.py +17 -2
  5. anemoi/datasets/commands/compare.py +18 -2
  6. anemoi/datasets/commands/copy.py +196 -14
  7. anemoi/datasets/commands/create.py +50 -7
  8. anemoi/datasets/commands/finalise-additions.py +17 -2
  9. anemoi/datasets/commands/finalise.py +17 -2
  10. anemoi/datasets/commands/init-additions.py +17 -2
  11. anemoi/datasets/commands/init.py +16 -2
  12. anemoi/datasets/commands/inspect.py +283 -62
  13. anemoi/datasets/commands/load-additions.py +16 -2
  14. anemoi/datasets/commands/load.py +16 -2
  15. anemoi/datasets/commands/patch.py +17 -2
  16. anemoi/datasets/commands/publish.py +17 -2
  17. anemoi/datasets/commands/scan.py +31 -3
  18. anemoi/datasets/compute/recentre.py +47 -11
  19. anemoi/datasets/create/__init__.py +612 -85
  20. anemoi/datasets/create/check.py +142 -20
  21. anemoi/datasets/create/chunks.py +64 -4
  22. anemoi/datasets/create/config.py +185 -21
  23. anemoi/datasets/create/filter.py +50 -0
  24. anemoi/datasets/create/filters/__init__.py +33 -0
  25. anemoi/datasets/create/filters/empty.py +37 -0
  26. anemoi/datasets/create/filters/legacy.py +93 -0
  27. anemoi/datasets/create/filters/noop.py +37 -0
  28. anemoi/datasets/create/filters/orog_to_z.py +58 -0
  29. anemoi/datasets/create/{functions/filters → filters}/pressure_level_relative_humidity_to_specific_humidity.py +33 -10
  30. anemoi/datasets/create/{functions/filters → filters}/pressure_level_specific_humidity_to_relative_humidity.py +32 -8
  31. anemoi/datasets/create/filters/rename.py +205 -0
  32. anemoi/datasets/create/{functions/filters → filters}/rotate_winds.py +43 -28
  33. anemoi/datasets/create/{functions/filters → filters}/single_level_dewpoint_to_relative_humidity.py +32 -9
  34. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_dewpoint.py +33 -9
  35. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_specific_humidity.py +55 -7
  36. anemoi/datasets/create/{functions/filters → filters}/single_level_specific_humidity_to_relative_humidity.py +98 -37
  37. anemoi/datasets/create/filters/speeddir_to_uv.py +95 -0
  38. anemoi/datasets/create/{functions/filters → filters}/sum.py +24 -27
  39. anemoi/datasets/create/filters/transform.py +53 -0
  40. anemoi/datasets/create/{functions/filters → filters}/unrotate_winds.py +27 -18
  41. anemoi/datasets/create/filters/uv_to_speeddir.py +94 -0
  42. anemoi/datasets/create/{functions/filters → filters}/wz_to_w.py +51 -33
  43. anemoi/datasets/create/input/__init__.py +76 -5
  44. anemoi/datasets/create/input/action.py +149 -13
  45. anemoi/datasets/create/input/concat.py +81 -10
  46. anemoi/datasets/create/input/context.py +39 -4
  47. anemoi/datasets/create/input/data_sources.py +72 -6
  48. anemoi/datasets/create/input/empty.py +21 -3
  49. anemoi/datasets/create/input/filter.py +60 -12
  50. anemoi/datasets/create/input/function.py +154 -37
  51. anemoi/datasets/create/input/join.py +86 -14
  52. anemoi/datasets/create/input/misc.py +67 -17
  53. anemoi/datasets/create/input/pipe.py +33 -6
  54. anemoi/datasets/create/input/repeated_dates.py +189 -41
  55. anemoi/datasets/create/input/result.py +202 -87
  56. anemoi/datasets/create/input/step.py +119 -22
  57. anemoi/datasets/create/input/template.py +100 -13
  58. anemoi/datasets/create/input/trace.py +62 -7
  59. anemoi/datasets/create/patch.py +52 -4
  60. anemoi/datasets/create/persistent.py +134 -17
  61. anemoi/datasets/create/size.py +15 -1
  62. anemoi/datasets/create/source.py +51 -0
  63. anemoi/datasets/create/sources/__init__.py +36 -0
  64. anemoi/datasets/create/{functions/sources → sources}/accumulations.py +296 -30
  65. anemoi/datasets/create/{functions/sources → sources}/constants.py +27 -2
  66. anemoi/datasets/create/{functions/sources → sources}/eccc_fstd.py +7 -3
  67. anemoi/datasets/create/sources/empty.py +37 -0
  68. anemoi/datasets/create/{functions/sources → sources}/forcings.py +25 -1
  69. anemoi/datasets/create/sources/grib.py +297 -0
  70. anemoi/datasets/create/{functions/sources → sources}/hindcasts.py +38 -4
  71. anemoi/datasets/create/sources/legacy.py +93 -0
  72. anemoi/datasets/create/{functions/sources → sources}/mars.py +168 -20
  73. anemoi/datasets/create/sources/netcdf.py +42 -0
  74. anemoi/datasets/create/sources/opendap.py +43 -0
  75. anemoi/datasets/create/{functions/sources/__init__.py → sources/patterns.py} +35 -4
  76. anemoi/datasets/create/sources/recentre.py +150 -0
  77. anemoi/datasets/create/{functions/sources → sources}/source.py +27 -5
  78. anemoi/datasets/create/{functions/sources → sources}/tendencies.py +64 -7
  79. anemoi/datasets/create/sources/xarray.py +92 -0
  80. anemoi/datasets/create/sources/xarray_kerchunk.py +36 -0
  81. anemoi/datasets/create/sources/xarray_support/README.md +1 -0
  82. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/__init__.py +109 -8
  83. anemoi/datasets/create/sources/xarray_support/coordinates.py +442 -0
  84. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/field.py +94 -16
  85. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/fieldlist.py +90 -25
  86. anemoi/datasets/create/sources/xarray_support/flavour.py +1036 -0
  87. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/grid.py +92 -31
  88. anemoi/datasets/create/sources/xarray_support/metadata.py +395 -0
  89. anemoi/datasets/create/sources/xarray_support/patch.py +91 -0
  90. anemoi/datasets/create/sources/xarray_support/time.py +391 -0
  91. anemoi/datasets/create/sources/xarray_support/variable.py +331 -0
  92. anemoi/datasets/create/sources/xarray_zarr.py +41 -0
  93. anemoi/datasets/create/{functions/sources → sources}/zenodo.py +34 -5
  94. anemoi/datasets/create/statistics/__init__.py +233 -44
  95. anemoi/datasets/create/statistics/summary.py +52 -6
  96. anemoi/datasets/create/testing.py +76 -0
  97. anemoi/datasets/create/{functions/filters/noop.py → typing.py} +6 -3
  98. anemoi/datasets/create/utils.py +97 -6
  99. anemoi/datasets/create/writer.py +26 -4
  100. anemoi/datasets/create/zarr.py +170 -23
  101. anemoi/datasets/data/__init__.py +51 -4
  102. anemoi/datasets/data/complement.py +191 -40
  103. anemoi/datasets/data/concat.py +141 -16
  104. anemoi/datasets/data/dataset.py +552 -61
  105. anemoi/datasets/data/debug.py +197 -26
  106. anemoi/datasets/data/ensemble.py +93 -8
  107. anemoi/datasets/data/fill_missing.py +165 -18
  108. anemoi/datasets/data/forwards.py +428 -56
  109. anemoi/datasets/data/grids.py +323 -97
  110. anemoi/datasets/data/indexing.py +112 -19
  111. anemoi/datasets/data/interpolate.py +92 -12
  112. anemoi/datasets/data/join.py +158 -19
  113. anemoi/datasets/data/masked.py +129 -15
  114. anemoi/datasets/data/merge.py +137 -23
  115. anemoi/datasets/data/misc.py +172 -16
  116. anemoi/datasets/data/missing.py +233 -29
  117. anemoi/datasets/data/rescale.py +111 -10
  118. anemoi/datasets/data/select.py +168 -26
  119. anemoi/datasets/data/statistics.py +67 -6
  120. anemoi/datasets/data/stores.py +149 -64
  121. anemoi/datasets/data/subset.py +159 -25
  122. anemoi/datasets/data/unchecked.py +168 -57
  123. anemoi/datasets/data/xy.py +168 -25
  124. anemoi/datasets/dates/__init__.py +191 -16
  125. anemoi/datasets/dates/groups.py +189 -47
  126. anemoi/datasets/grids.py +270 -31
  127. anemoi/datasets/testing.py +28 -1
  128. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/METADATA +9 -6
  129. anemoi_datasets-0.5.17.dist-info/RECORD +137 -0
  130. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/WHEEL +1 -1
  131. anemoi/datasets/create/functions/__init__.py +0 -66
  132. anemoi/datasets/create/functions/filters/__init__.py +0 -9
  133. anemoi/datasets/create/functions/filters/empty.py +0 -17
  134. anemoi/datasets/create/functions/filters/orog_to_z.py +0 -58
  135. anemoi/datasets/create/functions/filters/rename.py +0 -79
  136. anemoi/datasets/create/functions/filters/speeddir_to_uv.py +0 -78
  137. anemoi/datasets/create/functions/filters/uv_to_speeddir.py +0 -56
  138. anemoi/datasets/create/functions/sources/empty.py +0 -15
  139. anemoi/datasets/create/functions/sources/grib.py +0 -150
  140. anemoi/datasets/create/functions/sources/netcdf.py +0 -15
  141. anemoi/datasets/create/functions/sources/opendap.py +0 -15
  142. anemoi/datasets/create/functions/sources/recentre.py +0 -60
  143. anemoi/datasets/create/functions/sources/xarray/coordinates.py +0 -255
  144. anemoi/datasets/create/functions/sources/xarray/flavour.py +0 -472
  145. anemoi/datasets/create/functions/sources/xarray/metadata.py +0 -148
  146. anemoi/datasets/create/functions/sources/xarray/patch.py +0 -44
  147. anemoi/datasets/create/functions/sources/xarray/time.py +0 -177
  148. anemoi/datasets/create/functions/sources/xarray/variable.py +0 -188
  149. anemoi/datasets/create/functions/sources/xarray_kerchunk.py +0 -42
  150. anemoi/datasets/create/functions/sources/xarray_zarr.py +0 -15
  151. anemoi/datasets/utils/fields.py +0 -47
  152. anemoi_datasets-0.5.16.dist-info/RECORD +0 -129
  153. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/entry_points.txt +0 -0
  154. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info/licenses}/LICENSE +0 -0
  155. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/top_level.txt +0 -0
@@ -9,11 +9,19 @@
9
9
 
10
10
 
11
11
  import logging
12
+ from typing import Any
13
+ from typing import Dict
14
+ from typing import Optional
15
+ from typing import Set
12
16
 
13
17
  import numpy as np
18
+ from numpy.typing import NDArray
14
19
 
15
20
  from anemoi.datasets.data import MissingDateError
16
21
 
22
+ from .dataset import Dataset
23
+ from .dataset import FullIndex
24
+ from .dataset import TupleIndex
17
25
  from .debug import Node
18
26
  from .debug import debug_indexing
19
27
  from .forwards import Forwards
@@ -26,29 +34,74 @@ LOG = logging.getLogger(__name__)
26
34
 
27
35
 
28
36
  class MissingDatesFill(Forwards):
29
- def __init__(self, dataset):
37
+ """Class to handle filling missing dates in a dataset."""
38
+
39
+ def __init__(self, dataset: Dataset) -> None:
40
+ """Initialize the MissingDatesFill class.
41
+
42
+ Parameters
43
+ ----------
44
+ dataset : Dataset
45
+ The dataset with missing dates.
46
+ """
30
47
  super().__init__(dataset)
31
48
  self._missing = set(dataset.missing)
32
- self._warnings = set()
49
+ self._warnings: Set[int] = set()
33
50
 
34
51
  @debug_indexing
35
52
  @expand_list_indexing
36
- def _get_tuple(self, index):
53
+ def _get_tuple(self, index: TupleIndex) -> NDArray[Any]:
54
+ """Get a tuple of data for the given index.
55
+
56
+ Parameters
57
+ ----------
58
+ index : TupleIndex
59
+ The index to retrieve data for.
60
+
61
+ Returns
62
+ -------
63
+ NDArray[Any]
64
+ The data for the given index.
65
+ """
37
66
  index, changes = index_to_slices(index, self.shape)
38
67
  index, previous = update_tuple(index, 0, slice(None))
39
68
  result = self._get_slice(previous)
40
69
  return apply_index_to_slices_changes(result[index], changes)
41
70
 
42
- def _get_slice(self, s):
71
+ def _get_slice(self, s: slice) -> NDArray[Any]:
72
+ """Get a slice of data.
73
+
74
+ Parameters
75
+ ----------
76
+ s : slice
77
+ The slice to retrieve data for.
78
+
79
+ Returns
80
+ -------
81
+ NDArray[Any]
82
+ The data for the given slice.
83
+ """
43
84
  return np.stack([self[i] for i in range(*s.indices(self._len))])
44
85
 
45
86
  @property
46
- def missing(self):
87
+ def missing(self) -> Set[int]:
88
+ """Get the set of missing dates."""
47
89
  return set()
48
90
 
49
91
  @debug_indexing
50
- def __getitem__(self, n):
51
-
92
+ def __getitem__(self, n: FullIndex) -> NDArray[Any]:
93
+ """Get the data for the given index.
94
+
95
+ Parameters
96
+ ----------
97
+ n : FullIndex
98
+ The index to retrieve data for.
99
+
100
+ Returns
101
+ -------
102
+ NDArray[Any]
103
+ The data for the given index.
104
+ """
52
105
  try:
53
106
  return self.forward[n]
54
107
  except MissingDateError:
@@ -84,14 +137,39 @@ class MissingDatesFill(Forwards):
84
137
 
85
138
 
86
139
  class MissingDatesClosest(MissingDatesFill):
87
-
88
- def __init__(self, dataset, closest):
140
+ """Class to handle filling missing dates using the closest available date."""
141
+
142
+ def __init__(self, dataset: Any, closest: str) -> None:
143
+ """Initialize the MissingDatesClosest class.
144
+
145
+ Parameters
146
+ ----------
147
+ dataset : Any
148
+ The dataset with missing dates.
149
+ closest : str
150
+ The strategy to use for filling missing dates ('up' or 'down').
151
+ """
89
152
  super().__init__(dataset)
90
153
  self.closest = closest
91
154
  self._closest = {}
92
155
 
93
- def _fill_missing(self, n, a, b):
94
-
156
+ def _fill_missing(self, n: int, a: Optional[int], b: Optional[int]) -> NDArray[Any]:
157
+ """Fill the missing date at the given index.
158
+
159
+ Parameters
160
+ ----------
161
+ n : int
162
+ The index of the missing date.
163
+ a : Optional[int]
164
+ The previous available date index.
165
+ b : Optional[int]
166
+ The next available date index.
167
+
168
+ Returns
169
+ -------
170
+ NDArray[Any]
171
+ The filled data for the missing date.
172
+ """
95
173
  if n not in self._warnings:
96
174
  LOG.warning(f"Missing date at index {n} ({self.dates[n]})")
97
175
  if abs(n - a) == abs(b - n):
@@ -111,19 +189,58 @@ class MissingDatesClosest(MissingDatesFill):
111
189
 
112
190
  return self.forward[self._closest[n]]
113
191
 
114
- def subclass_metadata_specific(self):
192
+ def forwards_subclass_metadata_specific(self) -> Dict[str, Any]:
193
+ """Get metadata specific to the subclass.
194
+
195
+ Returns
196
+ -------
197
+ Dict[str, Any]
198
+ The metadata specific to the subclass.
199
+ """
115
200
  return {"closest": self.closest}
116
201
 
117
- def tree(self):
202
+ def tree(self) -> Node:
203
+ """Get the tree representation of the object.
204
+
205
+ Returns
206
+ -------
207
+ Node
208
+ The tree representation of the object.
209
+ """
118
210
  return Node(self, [self.forward.tree()], closest=self.closest)
119
211
 
120
212
 
121
213
  class MissingDatesInterpolate(MissingDatesFill):
122
- def __init__(self, dataset):
214
+ """Class to handle filling missing dates using interpolation."""
215
+
216
+ def __init__(self, dataset: Any) -> None:
217
+ """Initialize the MissingDatesInterpolate class.
218
+
219
+ Parameters
220
+ ----------
221
+ dataset : Any
222
+ The dataset with missing dates.
223
+ """
123
224
  super().__init__(dataset)
124
225
  self._alpha = {}
125
226
 
126
- def _fill_missing(self, n, a, b):
227
+ def _fill_missing(self, n: int, a: Optional[int], b: Optional[int]) -> NDArray[Any]:
228
+ """Fill the missing date at the given index using interpolation.
229
+
230
+ Parameters
231
+ ----------
232
+ n : int
233
+ The index of the missing date.
234
+ a : Optional[int]
235
+ The previous available date index.
236
+ b : Optional[int]
237
+ The next available date index.
238
+
239
+ Returns
240
+ -------
241
+ NDArray[Any]
242
+ The filled data for the missing date.
243
+ """
127
244
  if n not in self._warnings:
128
245
  LOG.warning(f"Missing date at index {n} ({self.dates[n]})")
129
246
 
@@ -147,14 +264,44 @@ class MissingDatesInterpolate(MissingDatesFill):
147
264
  alpha = self._alpha[n]
148
265
  return self.forward[a] * (1 - alpha) + self.forward[b] * alpha
149
266
 
150
- def subclass_metadata_specific(self):
267
+ def forwards_subclass_metadata_specific(self) -> Dict[str, Any]:
268
+ """Get metadata specific to the subclass.
269
+
270
+ Returns
271
+ -------
272
+ Dict[str, Any]
273
+ The metadata specific to the subclass.
274
+ """
151
275
  return {}
152
276
 
153
- def tree(self):
277
+ def tree(self) -> Node:
278
+ """Get the tree representation of the object.
279
+
280
+ Returns
281
+ -------
282
+ Node
283
+ The tree representation of the object.
284
+ """
154
285
  return Node(self, [self.forward.tree()])
155
286
 
156
287
 
157
- def fill_missing_dates_factory(dataset, method, kwargs):
288
+ def fill_missing_dates_factory(dataset: Any, method: str, kwargs: Dict[str, Any]) -> Dataset:
289
+ """Factory function to create an instance of a class to fill missing dates.
290
+
291
+ Parameters
292
+ ----------
293
+ dataset : Any
294
+ The dataset with missing dates.
295
+ method : str
296
+ The method to use for filling missing dates ('closest' or 'interpolate').
297
+ kwargs : Dict[str, Any]
298
+ Additional arguments for the method.
299
+
300
+ Returns
301
+ -------
302
+ Dataset
303
+ An instance of a class to fill missing dates.
304
+ """
158
305
  if method == "closest":
159
306
  closest = kwargs.get("closest", "up")
160
307
  return MissingDatesClosest(dataset, closest=closest)