anemoi-datasets 0.5.15__py3-none-any.whl → 0.5.17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. anemoi/datasets/__init__.py +4 -1
  2. anemoi/datasets/__main__.py +12 -2
  3. anemoi/datasets/_version.py +9 -4
  4. anemoi/datasets/commands/cleanup.py +17 -2
  5. anemoi/datasets/commands/compare.py +18 -2
  6. anemoi/datasets/commands/copy.py +196 -14
  7. anemoi/datasets/commands/create.py +50 -7
  8. anemoi/datasets/commands/finalise-additions.py +17 -2
  9. anemoi/datasets/commands/finalise.py +17 -2
  10. anemoi/datasets/commands/init-additions.py +17 -2
  11. anemoi/datasets/commands/init.py +16 -2
  12. anemoi/datasets/commands/inspect.py +283 -62
  13. anemoi/datasets/commands/load-additions.py +16 -2
  14. anemoi/datasets/commands/load.py +16 -2
  15. anemoi/datasets/commands/patch.py +17 -2
  16. anemoi/datasets/commands/publish.py +17 -2
  17. anemoi/datasets/commands/scan.py +31 -3
  18. anemoi/datasets/compute/recentre.py +47 -11
  19. anemoi/datasets/create/__init__.py +612 -85
  20. anemoi/datasets/create/check.py +142 -20
  21. anemoi/datasets/create/chunks.py +64 -4
  22. anemoi/datasets/create/config.py +185 -21
  23. anemoi/datasets/create/filter.py +50 -0
  24. anemoi/datasets/create/filters/__init__.py +33 -0
  25. anemoi/datasets/create/filters/empty.py +37 -0
  26. anemoi/datasets/create/filters/legacy.py +93 -0
  27. anemoi/datasets/create/filters/noop.py +37 -0
  28. anemoi/datasets/create/filters/orog_to_z.py +58 -0
  29. anemoi/datasets/create/{functions/filters → filters}/pressure_level_relative_humidity_to_specific_humidity.py +33 -10
  30. anemoi/datasets/create/{functions/filters → filters}/pressure_level_specific_humidity_to_relative_humidity.py +32 -8
  31. anemoi/datasets/create/filters/rename.py +205 -0
  32. anemoi/datasets/create/{functions/filters → filters}/rotate_winds.py +43 -28
  33. anemoi/datasets/create/{functions/filters → filters}/single_level_dewpoint_to_relative_humidity.py +32 -9
  34. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_dewpoint.py +33 -9
  35. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_specific_humidity.py +55 -7
  36. anemoi/datasets/create/{functions/filters → filters}/single_level_specific_humidity_to_relative_humidity.py +98 -37
  37. anemoi/datasets/create/filters/speeddir_to_uv.py +95 -0
  38. anemoi/datasets/create/{functions/filters → filters}/sum.py +24 -27
  39. anemoi/datasets/create/filters/transform.py +53 -0
  40. anemoi/datasets/create/{functions/filters → filters}/unrotate_winds.py +27 -18
  41. anemoi/datasets/create/filters/uv_to_speeddir.py +94 -0
  42. anemoi/datasets/create/{functions/filters → filters}/wz_to_w.py +51 -33
  43. anemoi/datasets/create/input/__init__.py +76 -5
  44. anemoi/datasets/create/input/action.py +149 -13
  45. anemoi/datasets/create/input/concat.py +81 -10
  46. anemoi/datasets/create/input/context.py +39 -4
  47. anemoi/datasets/create/input/data_sources.py +72 -6
  48. anemoi/datasets/create/input/empty.py +21 -3
  49. anemoi/datasets/create/input/filter.py +60 -12
  50. anemoi/datasets/create/input/function.py +154 -37
  51. anemoi/datasets/create/input/join.py +86 -14
  52. anemoi/datasets/create/input/misc.py +67 -17
  53. anemoi/datasets/create/input/pipe.py +33 -6
  54. anemoi/datasets/create/input/repeated_dates.py +189 -41
  55. anemoi/datasets/create/input/result.py +202 -87
  56. anemoi/datasets/create/input/step.py +119 -22
  57. anemoi/datasets/create/input/template.py +100 -13
  58. anemoi/datasets/create/input/trace.py +62 -7
  59. anemoi/datasets/create/patch.py +52 -4
  60. anemoi/datasets/create/persistent.py +134 -17
  61. anemoi/datasets/create/size.py +15 -1
  62. anemoi/datasets/create/source.py +51 -0
  63. anemoi/datasets/create/sources/__init__.py +36 -0
  64. anemoi/datasets/create/{functions/sources → sources}/accumulations.py +296 -30
  65. anemoi/datasets/create/{functions/sources → sources}/constants.py +27 -2
  66. anemoi/datasets/create/{functions/sources → sources}/eccc_fstd.py +7 -3
  67. anemoi/datasets/create/sources/empty.py +37 -0
  68. anemoi/datasets/create/{functions/sources → sources}/forcings.py +25 -1
  69. anemoi/datasets/create/sources/grib.py +297 -0
  70. anemoi/datasets/create/{functions/sources → sources}/hindcasts.py +38 -4
  71. anemoi/datasets/create/sources/legacy.py +93 -0
  72. anemoi/datasets/create/{functions/sources → sources}/mars.py +168 -20
  73. anemoi/datasets/create/sources/netcdf.py +42 -0
  74. anemoi/datasets/create/sources/opendap.py +43 -0
  75. anemoi/datasets/create/{functions/sources/__init__.py → sources/patterns.py} +35 -4
  76. anemoi/datasets/create/sources/recentre.py +150 -0
  77. anemoi/datasets/create/{functions/sources → sources}/source.py +27 -5
  78. anemoi/datasets/create/{functions/sources → sources}/tendencies.py +64 -7
  79. anemoi/datasets/create/sources/xarray.py +92 -0
  80. anemoi/datasets/create/sources/xarray_kerchunk.py +36 -0
  81. anemoi/datasets/create/sources/xarray_support/README.md +1 -0
  82. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/__init__.py +109 -8
  83. anemoi/datasets/create/sources/xarray_support/coordinates.py +442 -0
  84. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/field.py +94 -16
  85. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/fieldlist.py +90 -25
  86. anemoi/datasets/create/sources/xarray_support/flavour.py +1036 -0
  87. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/grid.py +92 -31
  88. anemoi/datasets/create/sources/xarray_support/metadata.py +395 -0
  89. anemoi/datasets/create/sources/xarray_support/patch.py +91 -0
  90. anemoi/datasets/create/sources/xarray_support/time.py +391 -0
  91. anemoi/datasets/create/sources/xarray_support/variable.py +331 -0
  92. anemoi/datasets/create/sources/xarray_zarr.py +41 -0
  93. anemoi/datasets/create/{functions/sources → sources}/zenodo.py +34 -5
  94. anemoi/datasets/create/statistics/__init__.py +233 -44
  95. anemoi/datasets/create/statistics/summary.py +52 -6
  96. anemoi/datasets/create/testing.py +76 -0
  97. anemoi/datasets/create/{functions/filters/noop.py → typing.py} +6 -3
  98. anemoi/datasets/create/utils.py +97 -6
  99. anemoi/datasets/create/writer.py +26 -4
  100. anemoi/datasets/create/zarr.py +170 -23
  101. anemoi/datasets/data/__init__.py +51 -4
  102. anemoi/datasets/data/complement.py +191 -40
  103. anemoi/datasets/data/concat.py +141 -16
  104. anemoi/datasets/data/dataset.py +552 -61
  105. anemoi/datasets/data/debug.py +197 -26
  106. anemoi/datasets/data/ensemble.py +93 -8
  107. anemoi/datasets/data/fill_missing.py +165 -18
  108. anemoi/datasets/data/forwards.py +428 -56
  109. anemoi/datasets/data/grids.py +323 -97
  110. anemoi/datasets/data/indexing.py +112 -19
  111. anemoi/datasets/data/interpolate.py +92 -12
  112. anemoi/datasets/data/join.py +158 -19
  113. anemoi/datasets/data/masked.py +129 -15
  114. anemoi/datasets/data/merge.py +137 -23
  115. anemoi/datasets/data/misc.py +172 -16
  116. anemoi/datasets/data/missing.py +233 -29
  117. anemoi/datasets/data/rescale.py +111 -10
  118. anemoi/datasets/data/select.py +168 -26
  119. anemoi/datasets/data/statistics.py +67 -6
  120. anemoi/datasets/data/stores.py +149 -64
  121. anemoi/datasets/data/subset.py +159 -25
  122. anemoi/datasets/data/unchecked.py +168 -57
  123. anemoi/datasets/data/xy.py +168 -25
  124. anemoi/datasets/dates/__init__.py +191 -16
  125. anemoi/datasets/dates/groups.py +189 -47
  126. anemoi/datasets/grids.py +270 -31
  127. anemoi/datasets/testing.py +28 -1
  128. {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/METADATA +10 -7
  129. anemoi_datasets-0.5.17.dist-info/RECORD +137 -0
  130. {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/WHEEL +1 -1
  131. {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info/licenses}/LICENSE +1 -1
  132. anemoi/datasets/create/functions/__init__.py +0 -66
  133. anemoi/datasets/create/functions/filters/__init__.py +0 -9
  134. anemoi/datasets/create/functions/filters/empty.py +0 -17
  135. anemoi/datasets/create/functions/filters/orog_to_z.py +0 -58
  136. anemoi/datasets/create/functions/filters/rename.py +0 -79
  137. anemoi/datasets/create/functions/filters/speeddir_to_uv.py +0 -78
  138. anemoi/datasets/create/functions/filters/uv_to_speeddir.py +0 -56
  139. anemoi/datasets/create/functions/sources/empty.py +0 -15
  140. anemoi/datasets/create/functions/sources/grib.py +0 -150
  141. anemoi/datasets/create/functions/sources/netcdf.py +0 -15
  142. anemoi/datasets/create/functions/sources/opendap.py +0 -15
  143. anemoi/datasets/create/functions/sources/recentre.py +0 -60
  144. anemoi/datasets/create/functions/sources/xarray/coordinates.py +0 -255
  145. anemoi/datasets/create/functions/sources/xarray/flavour.py +0 -472
  146. anemoi/datasets/create/functions/sources/xarray/metadata.py +0 -148
  147. anemoi/datasets/create/functions/sources/xarray/patch.py +0 -44
  148. anemoi/datasets/create/functions/sources/xarray/time.py +0 -177
  149. anemoi/datasets/create/functions/sources/xarray/variable.py +0 -188
  150. anemoi/datasets/create/functions/sources/xarray_kerchunk.py +0 -42
  151. anemoi/datasets/create/functions/sources/xarray_zarr.py +0 -15
  152. anemoi/datasets/utils/fields.py +0 -47
  153. anemoi_datasets-0.5.15.dist-info/RECORD +0 -129
  154. {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/entry_points.txt +0 -0
  155. {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/top_level.txt +0 -0
@@ -1,255 +0,0 @@
1
- # (C) Copyright 2024 Anemoi contributors.
2
- #
3
- # This software is licensed under the terms of the Apache Licence Version 2.0
4
- # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
- #
6
- # In applying this licence, ECMWF does not waive the privileges and immunities
7
- # granted to it by virtue of its status as an intergovernmental organisation
8
- # nor does it submit to any jurisdiction.
9
-
10
-
11
- import datetime
12
- import logging
13
-
14
- import numpy as np
15
- from earthkit.data.utils.dates import to_datetime
16
-
17
- LOG = logging.getLogger(__name__)
18
-
19
-
20
- def is_scalar(variable):
21
- shape = variable.shape
22
- if shape == (1,):
23
- return True
24
- if len(shape) == 0:
25
- return True
26
- return False
27
-
28
-
29
- def extract_single_value(variable):
30
- shape = variable.shape
31
- if np.issubdtype(variable.values.dtype, np.datetime64):
32
- if len(shape) == 0:
33
- return to_datetime(variable.values) # Convert to python datetime
34
- if shape == (1,):
35
- return to_datetime(variable.values[0])
36
- assert False, (shape, variable.values[:2])
37
-
38
- if np.issubdtype(variable.values.dtype, np.timedelta64):
39
- if len(shape) == 0:
40
- # Convert to python timedelta64
41
- return datetime.timedelta(seconds=variable.values.astype("timedelta64[s]").astype(int).item())
42
- assert False, (shape, variable.values)
43
-
44
- if shape == (1,):
45
- return variable.values[0]
46
-
47
- if len(shape) == 0:
48
- return variable.values.item()
49
-
50
- assert False, (shape, variable.values)
51
-
52
-
53
- class Coordinate:
54
- is_grid = False
55
- is_dim = True
56
- is_lat = False
57
- is_lon = False
58
- is_time = False
59
- is_step = False
60
- is_date = False
61
- is_member = False
62
- is_x = False
63
- is_y = False
64
-
65
- def __init__(self, variable):
66
- self.variable = variable
67
- self.scalar = is_scalar(variable)
68
- self.kwargs = {} # Used when creating a new coordinate (reduced method)
69
-
70
- def __len__(self):
71
- return 1 if self.scalar else len(self.variable)
72
-
73
- def __repr__(self):
74
- return "%s[name=%s,values=%s,shape=%s]" % (
75
- self.__class__.__name__,
76
- self.variable.name,
77
- self.variable.values if self.scalar else len(self),
78
- self.variable.shape,
79
- )
80
-
81
- def reduced(self, i):
82
- """Create a new coordinate with a single value
83
-
84
- Parameters
85
- ----------
86
- i : int
87
- the index of the value to select
88
-
89
- Returns
90
- -------
91
- Coordinate
92
- the new coordinate
93
- """
94
- return self.__class__(
95
- self.variable.isel({self.variable.dims[0]: i}),
96
- **self.kwargs,
97
- )
98
-
99
- def index(self, value):
100
- """Return the index of the value in the coordinate
101
-
102
- Parameters
103
- ----------
104
- value : Any
105
- The value to search for
106
-
107
- Returns
108
- -------
109
- int or None
110
- The index of the value in the coordinate or None if not found
111
- """
112
-
113
- if isinstance(value, (list, tuple)):
114
- if len(value) == 1:
115
- return self._index_single(value)
116
- else:
117
- return self._index_multiple(value)
118
- return self._index_single(value)
119
-
120
- def _index_single(self, value):
121
-
122
- values = self.variable.values
123
-
124
- # Assume the array is sorted
125
- index = np.searchsorted(values, value)
126
-
127
- if index < len(values) and values[index] == value:
128
- return index
129
-
130
- # If not found, we need to check if the value is in the array
131
-
132
- index = np.where(values == value)[0]
133
- if len(index) > 0:
134
- return index[0]
135
-
136
- return None
137
-
138
- def _index_multiple(self, value):
139
-
140
- values = self.variable.values
141
-
142
- # Assume the array is sorted
143
-
144
- index = np.searchsorted(values, value)
145
- index = index[index < len(values)]
146
-
147
- if np.all(values[index] == value):
148
- return index
149
-
150
- # If not found, we need to check if the value is in the array
151
-
152
- index = np.where(np.isin(values, value))[0]
153
-
154
- # We could also return incomplete matches
155
- if len(index) == len(value):
156
- return index
157
-
158
- return None
159
-
160
- @property
161
- def name(self):
162
- return self.variable.name
163
-
164
- def normalise(self, value):
165
- # Subclasses to format values that will be added to the field metadata
166
- return value
167
-
168
- @property
169
- def single_value(self):
170
- return extract_single_value(self.variable)
171
-
172
-
173
- class TimeCoordinate(Coordinate):
174
- is_time = True
175
- mars_names = ("valid_datetime",)
176
-
177
- def index(self, time):
178
- return super().index(np.datetime64(time))
179
-
180
-
181
- class DateCoordinate(Coordinate):
182
- is_date = True
183
- mars_names = ("date",)
184
-
185
- def index(self, date):
186
- return super().index(np.datetime64(date))
187
-
188
-
189
- class StepCoordinate(Coordinate):
190
- is_step = True
191
- mars_names = ("step",)
192
-
193
-
194
- class LevelCoordinate(Coordinate):
195
- mars_names = ("level", "levelist")
196
-
197
- def __init__(self, variable, levtype):
198
- super().__init__(variable)
199
- self.levtype = levtype
200
- # kwargs is used when creating a new coordinate (reduced method)
201
- self.kwargs = {"levtype": levtype}
202
-
203
- def normalise(self, value):
204
- # Some netcdf have pressue levels in float
205
- if int(value) == value:
206
- return int(value)
207
- return value
208
-
209
-
210
- class EnsembleCoordinate(Coordinate):
211
- is_member = True
212
- mars_names = ("number",)
213
-
214
- def normalise(self, value):
215
- if int(value) == value:
216
- return int(value)
217
- return value
218
-
219
-
220
- class LongitudeCoordinate(Coordinate):
221
- is_grid = True
222
- is_lon = True
223
- mars_names = ("longitude",)
224
-
225
-
226
- class LatitudeCoordinate(Coordinate):
227
- is_grid = True
228
- is_lat = True
229
- mars_names = ("latitude",)
230
-
231
-
232
- class XCoordinate(Coordinate):
233
- is_grid = True
234
- is_x = True
235
- mars_names = ("x",)
236
-
237
-
238
- class YCoordinate(Coordinate):
239
- is_grid = True
240
- is_y = True
241
- mars_names = ("y",)
242
-
243
-
244
- class ScalarCoordinate(Coordinate):
245
- is_grid = False
246
-
247
- @property
248
- def mars_names(self):
249
- return (self.variable.name,)
250
-
251
-
252
- class UnsupportedCoordinate(Coordinate):
253
- @property
254
- def mars_names(self):
255
- return (self.variable.name,)