anemoi-datasets 0.5.15__py3-none-any.whl → 0.5.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anemoi/datasets/__init__.py +4 -1
- anemoi/datasets/__main__.py +12 -2
- anemoi/datasets/_version.py +9 -4
- anemoi/datasets/commands/cleanup.py +17 -2
- anemoi/datasets/commands/compare.py +18 -2
- anemoi/datasets/commands/copy.py +196 -14
- anemoi/datasets/commands/create.py +50 -7
- anemoi/datasets/commands/finalise-additions.py +17 -2
- anemoi/datasets/commands/finalise.py +17 -2
- anemoi/datasets/commands/init-additions.py +17 -2
- anemoi/datasets/commands/init.py +16 -2
- anemoi/datasets/commands/inspect.py +283 -62
- anemoi/datasets/commands/load-additions.py +16 -2
- anemoi/datasets/commands/load.py +16 -2
- anemoi/datasets/commands/patch.py +17 -2
- anemoi/datasets/commands/publish.py +17 -2
- anemoi/datasets/commands/scan.py +31 -3
- anemoi/datasets/compute/recentre.py +47 -11
- anemoi/datasets/create/__init__.py +612 -85
- anemoi/datasets/create/check.py +142 -20
- anemoi/datasets/create/chunks.py +64 -4
- anemoi/datasets/create/config.py +185 -21
- anemoi/datasets/create/filter.py +50 -0
- anemoi/datasets/create/filters/__init__.py +33 -0
- anemoi/datasets/create/filters/empty.py +37 -0
- anemoi/datasets/create/filters/legacy.py +93 -0
- anemoi/datasets/create/filters/noop.py +37 -0
- anemoi/datasets/create/filters/orog_to_z.py +58 -0
- anemoi/datasets/create/{functions/filters → filters}/pressure_level_relative_humidity_to_specific_humidity.py +33 -10
- anemoi/datasets/create/{functions/filters → filters}/pressure_level_specific_humidity_to_relative_humidity.py +32 -8
- anemoi/datasets/create/filters/rename.py +205 -0
- anemoi/datasets/create/{functions/filters → filters}/rotate_winds.py +43 -28
- anemoi/datasets/create/{functions/filters → filters}/single_level_dewpoint_to_relative_humidity.py +32 -9
- anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_dewpoint.py +33 -9
- anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_specific_humidity.py +55 -7
- anemoi/datasets/create/{functions/filters → filters}/single_level_specific_humidity_to_relative_humidity.py +98 -37
- anemoi/datasets/create/filters/speeddir_to_uv.py +95 -0
- anemoi/datasets/create/{functions/filters → filters}/sum.py +24 -27
- anemoi/datasets/create/filters/transform.py +53 -0
- anemoi/datasets/create/{functions/filters → filters}/unrotate_winds.py +27 -18
- anemoi/datasets/create/filters/uv_to_speeddir.py +94 -0
- anemoi/datasets/create/{functions/filters → filters}/wz_to_w.py +51 -33
- anemoi/datasets/create/input/__init__.py +76 -5
- anemoi/datasets/create/input/action.py +149 -13
- anemoi/datasets/create/input/concat.py +81 -10
- anemoi/datasets/create/input/context.py +39 -4
- anemoi/datasets/create/input/data_sources.py +72 -6
- anemoi/datasets/create/input/empty.py +21 -3
- anemoi/datasets/create/input/filter.py +60 -12
- anemoi/datasets/create/input/function.py +154 -37
- anemoi/datasets/create/input/join.py +86 -14
- anemoi/datasets/create/input/misc.py +67 -17
- anemoi/datasets/create/input/pipe.py +33 -6
- anemoi/datasets/create/input/repeated_dates.py +189 -41
- anemoi/datasets/create/input/result.py +202 -87
- anemoi/datasets/create/input/step.py +119 -22
- anemoi/datasets/create/input/template.py +100 -13
- anemoi/datasets/create/input/trace.py +62 -7
- anemoi/datasets/create/patch.py +52 -4
- anemoi/datasets/create/persistent.py +134 -17
- anemoi/datasets/create/size.py +15 -1
- anemoi/datasets/create/source.py +51 -0
- anemoi/datasets/create/sources/__init__.py +36 -0
- anemoi/datasets/create/{functions/sources → sources}/accumulations.py +296 -30
- anemoi/datasets/create/{functions/sources → sources}/constants.py +27 -2
- anemoi/datasets/create/{functions/sources → sources}/eccc_fstd.py +7 -3
- anemoi/datasets/create/sources/empty.py +37 -0
- anemoi/datasets/create/{functions/sources → sources}/forcings.py +25 -1
- anemoi/datasets/create/sources/grib.py +297 -0
- anemoi/datasets/create/{functions/sources → sources}/hindcasts.py +38 -4
- anemoi/datasets/create/sources/legacy.py +93 -0
- anemoi/datasets/create/{functions/sources → sources}/mars.py +168 -20
- anemoi/datasets/create/sources/netcdf.py +42 -0
- anemoi/datasets/create/sources/opendap.py +43 -0
- anemoi/datasets/create/{functions/sources/__init__.py → sources/patterns.py} +35 -4
- anemoi/datasets/create/sources/recentre.py +150 -0
- anemoi/datasets/create/{functions/sources → sources}/source.py +27 -5
- anemoi/datasets/create/{functions/sources → sources}/tendencies.py +64 -7
- anemoi/datasets/create/sources/xarray.py +92 -0
- anemoi/datasets/create/sources/xarray_kerchunk.py +36 -0
- anemoi/datasets/create/sources/xarray_support/README.md +1 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/__init__.py +109 -8
- anemoi/datasets/create/sources/xarray_support/coordinates.py +442 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/field.py +94 -16
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/fieldlist.py +90 -25
- anemoi/datasets/create/sources/xarray_support/flavour.py +1036 -0
- anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/grid.py +92 -31
- anemoi/datasets/create/sources/xarray_support/metadata.py +395 -0
- anemoi/datasets/create/sources/xarray_support/patch.py +91 -0
- anemoi/datasets/create/sources/xarray_support/time.py +391 -0
- anemoi/datasets/create/sources/xarray_support/variable.py +331 -0
- anemoi/datasets/create/sources/xarray_zarr.py +41 -0
- anemoi/datasets/create/{functions/sources → sources}/zenodo.py +34 -5
- anemoi/datasets/create/statistics/__init__.py +233 -44
- anemoi/datasets/create/statistics/summary.py +52 -6
- anemoi/datasets/create/testing.py +76 -0
- anemoi/datasets/create/{functions/filters/noop.py → typing.py} +6 -3
- anemoi/datasets/create/utils.py +97 -6
- anemoi/datasets/create/writer.py +26 -4
- anemoi/datasets/create/zarr.py +170 -23
- anemoi/datasets/data/__init__.py +51 -4
- anemoi/datasets/data/complement.py +191 -40
- anemoi/datasets/data/concat.py +141 -16
- anemoi/datasets/data/dataset.py +552 -61
- anemoi/datasets/data/debug.py +197 -26
- anemoi/datasets/data/ensemble.py +93 -8
- anemoi/datasets/data/fill_missing.py +165 -18
- anemoi/datasets/data/forwards.py +428 -56
- anemoi/datasets/data/grids.py +323 -97
- anemoi/datasets/data/indexing.py +112 -19
- anemoi/datasets/data/interpolate.py +92 -12
- anemoi/datasets/data/join.py +158 -19
- anemoi/datasets/data/masked.py +129 -15
- anemoi/datasets/data/merge.py +137 -23
- anemoi/datasets/data/misc.py +172 -16
- anemoi/datasets/data/missing.py +233 -29
- anemoi/datasets/data/rescale.py +111 -10
- anemoi/datasets/data/select.py +168 -26
- anemoi/datasets/data/statistics.py +67 -6
- anemoi/datasets/data/stores.py +149 -64
- anemoi/datasets/data/subset.py +159 -25
- anemoi/datasets/data/unchecked.py +168 -57
- anemoi/datasets/data/xy.py +168 -25
- anemoi/datasets/dates/__init__.py +191 -16
- anemoi/datasets/dates/groups.py +189 -47
- anemoi/datasets/grids.py +270 -31
- anemoi/datasets/testing.py +28 -1
- {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/METADATA +10 -7
- anemoi_datasets-0.5.17.dist-info/RECORD +137 -0
- {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/WHEEL +1 -1
- {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info/licenses}/LICENSE +1 -1
- anemoi/datasets/create/functions/__init__.py +0 -66
- anemoi/datasets/create/functions/filters/__init__.py +0 -9
- anemoi/datasets/create/functions/filters/empty.py +0 -17
- anemoi/datasets/create/functions/filters/orog_to_z.py +0 -58
- anemoi/datasets/create/functions/filters/rename.py +0 -79
- anemoi/datasets/create/functions/filters/speeddir_to_uv.py +0 -78
- anemoi/datasets/create/functions/filters/uv_to_speeddir.py +0 -56
- anemoi/datasets/create/functions/sources/empty.py +0 -15
- anemoi/datasets/create/functions/sources/grib.py +0 -150
- anemoi/datasets/create/functions/sources/netcdf.py +0 -15
- anemoi/datasets/create/functions/sources/opendap.py +0 -15
- anemoi/datasets/create/functions/sources/recentre.py +0 -60
- anemoi/datasets/create/functions/sources/xarray/coordinates.py +0 -255
- anemoi/datasets/create/functions/sources/xarray/flavour.py +0 -472
- anemoi/datasets/create/functions/sources/xarray/metadata.py +0 -148
- anemoi/datasets/create/functions/sources/xarray/patch.py +0 -44
- anemoi/datasets/create/functions/sources/xarray/time.py +0 -177
- anemoi/datasets/create/functions/sources/xarray/variable.py +0 -188
- anemoi/datasets/create/functions/sources/xarray_kerchunk.py +0 -42
- anemoi/datasets/create/functions/sources/xarray_zarr.py +0 -15
- anemoi/datasets/utils/fields.py +0 -47
- anemoi_datasets-0.5.15.dist-info/RECORD +0 -129
- {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/entry_points.txt +0 -0
- {anemoi_datasets-0.5.15.dist-info → anemoi_datasets-0.5.17.dist-info}/top_level.txt +0 -0
|
@@ -1,255 +0,0 @@
|
|
|
1
|
-
# (C) Copyright 2024 Anemoi contributors.
|
|
2
|
-
#
|
|
3
|
-
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
|
-
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
-
#
|
|
6
|
-
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
-
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
-
# nor does it submit to any jurisdiction.
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
import datetime
|
|
12
|
-
import logging
|
|
13
|
-
|
|
14
|
-
import numpy as np
|
|
15
|
-
from earthkit.data.utils.dates import to_datetime
|
|
16
|
-
|
|
17
|
-
LOG = logging.getLogger(__name__)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def is_scalar(variable):
|
|
21
|
-
shape = variable.shape
|
|
22
|
-
if shape == (1,):
|
|
23
|
-
return True
|
|
24
|
-
if len(shape) == 0:
|
|
25
|
-
return True
|
|
26
|
-
return False
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def extract_single_value(variable):
|
|
30
|
-
shape = variable.shape
|
|
31
|
-
if np.issubdtype(variable.values.dtype, np.datetime64):
|
|
32
|
-
if len(shape) == 0:
|
|
33
|
-
return to_datetime(variable.values) # Convert to python datetime
|
|
34
|
-
if shape == (1,):
|
|
35
|
-
return to_datetime(variable.values[0])
|
|
36
|
-
assert False, (shape, variable.values[:2])
|
|
37
|
-
|
|
38
|
-
if np.issubdtype(variable.values.dtype, np.timedelta64):
|
|
39
|
-
if len(shape) == 0:
|
|
40
|
-
# Convert to python timedelta64
|
|
41
|
-
return datetime.timedelta(seconds=variable.values.astype("timedelta64[s]").astype(int).item())
|
|
42
|
-
assert False, (shape, variable.values)
|
|
43
|
-
|
|
44
|
-
if shape == (1,):
|
|
45
|
-
return variable.values[0]
|
|
46
|
-
|
|
47
|
-
if len(shape) == 0:
|
|
48
|
-
return variable.values.item()
|
|
49
|
-
|
|
50
|
-
assert False, (shape, variable.values)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class Coordinate:
|
|
54
|
-
is_grid = False
|
|
55
|
-
is_dim = True
|
|
56
|
-
is_lat = False
|
|
57
|
-
is_lon = False
|
|
58
|
-
is_time = False
|
|
59
|
-
is_step = False
|
|
60
|
-
is_date = False
|
|
61
|
-
is_member = False
|
|
62
|
-
is_x = False
|
|
63
|
-
is_y = False
|
|
64
|
-
|
|
65
|
-
def __init__(self, variable):
|
|
66
|
-
self.variable = variable
|
|
67
|
-
self.scalar = is_scalar(variable)
|
|
68
|
-
self.kwargs = {} # Used when creating a new coordinate (reduced method)
|
|
69
|
-
|
|
70
|
-
def __len__(self):
|
|
71
|
-
return 1 if self.scalar else len(self.variable)
|
|
72
|
-
|
|
73
|
-
def __repr__(self):
|
|
74
|
-
return "%s[name=%s,values=%s,shape=%s]" % (
|
|
75
|
-
self.__class__.__name__,
|
|
76
|
-
self.variable.name,
|
|
77
|
-
self.variable.values if self.scalar else len(self),
|
|
78
|
-
self.variable.shape,
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
def reduced(self, i):
|
|
82
|
-
"""Create a new coordinate with a single value
|
|
83
|
-
|
|
84
|
-
Parameters
|
|
85
|
-
----------
|
|
86
|
-
i : int
|
|
87
|
-
the index of the value to select
|
|
88
|
-
|
|
89
|
-
Returns
|
|
90
|
-
-------
|
|
91
|
-
Coordinate
|
|
92
|
-
the new coordinate
|
|
93
|
-
"""
|
|
94
|
-
return self.__class__(
|
|
95
|
-
self.variable.isel({self.variable.dims[0]: i}),
|
|
96
|
-
**self.kwargs,
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
def index(self, value):
|
|
100
|
-
"""Return the index of the value in the coordinate
|
|
101
|
-
|
|
102
|
-
Parameters
|
|
103
|
-
----------
|
|
104
|
-
value : Any
|
|
105
|
-
The value to search for
|
|
106
|
-
|
|
107
|
-
Returns
|
|
108
|
-
-------
|
|
109
|
-
int or None
|
|
110
|
-
The index of the value in the coordinate or None if not found
|
|
111
|
-
"""
|
|
112
|
-
|
|
113
|
-
if isinstance(value, (list, tuple)):
|
|
114
|
-
if len(value) == 1:
|
|
115
|
-
return self._index_single(value)
|
|
116
|
-
else:
|
|
117
|
-
return self._index_multiple(value)
|
|
118
|
-
return self._index_single(value)
|
|
119
|
-
|
|
120
|
-
def _index_single(self, value):
|
|
121
|
-
|
|
122
|
-
values = self.variable.values
|
|
123
|
-
|
|
124
|
-
# Assume the array is sorted
|
|
125
|
-
index = np.searchsorted(values, value)
|
|
126
|
-
|
|
127
|
-
if index < len(values) and values[index] == value:
|
|
128
|
-
return index
|
|
129
|
-
|
|
130
|
-
# If not found, we need to check if the value is in the array
|
|
131
|
-
|
|
132
|
-
index = np.where(values == value)[0]
|
|
133
|
-
if len(index) > 0:
|
|
134
|
-
return index[0]
|
|
135
|
-
|
|
136
|
-
return None
|
|
137
|
-
|
|
138
|
-
def _index_multiple(self, value):
|
|
139
|
-
|
|
140
|
-
values = self.variable.values
|
|
141
|
-
|
|
142
|
-
# Assume the array is sorted
|
|
143
|
-
|
|
144
|
-
index = np.searchsorted(values, value)
|
|
145
|
-
index = index[index < len(values)]
|
|
146
|
-
|
|
147
|
-
if np.all(values[index] == value):
|
|
148
|
-
return index
|
|
149
|
-
|
|
150
|
-
# If not found, we need to check if the value is in the array
|
|
151
|
-
|
|
152
|
-
index = np.where(np.isin(values, value))[0]
|
|
153
|
-
|
|
154
|
-
# We could also return incomplete matches
|
|
155
|
-
if len(index) == len(value):
|
|
156
|
-
return index
|
|
157
|
-
|
|
158
|
-
return None
|
|
159
|
-
|
|
160
|
-
@property
|
|
161
|
-
def name(self):
|
|
162
|
-
return self.variable.name
|
|
163
|
-
|
|
164
|
-
def normalise(self, value):
|
|
165
|
-
# Subclasses to format values that will be added to the field metadata
|
|
166
|
-
return value
|
|
167
|
-
|
|
168
|
-
@property
|
|
169
|
-
def single_value(self):
|
|
170
|
-
return extract_single_value(self.variable)
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
class TimeCoordinate(Coordinate):
|
|
174
|
-
is_time = True
|
|
175
|
-
mars_names = ("valid_datetime",)
|
|
176
|
-
|
|
177
|
-
def index(self, time):
|
|
178
|
-
return super().index(np.datetime64(time))
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
class DateCoordinate(Coordinate):
|
|
182
|
-
is_date = True
|
|
183
|
-
mars_names = ("date",)
|
|
184
|
-
|
|
185
|
-
def index(self, date):
|
|
186
|
-
return super().index(np.datetime64(date))
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
class StepCoordinate(Coordinate):
|
|
190
|
-
is_step = True
|
|
191
|
-
mars_names = ("step",)
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
class LevelCoordinate(Coordinate):
|
|
195
|
-
mars_names = ("level", "levelist")
|
|
196
|
-
|
|
197
|
-
def __init__(self, variable, levtype):
|
|
198
|
-
super().__init__(variable)
|
|
199
|
-
self.levtype = levtype
|
|
200
|
-
# kwargs is used when creating a new coordinate (reduced method)
|
|
201
|
-
self.kwargs = {"levtype": levtype}
|
|
202
|
-
|
|
203
|
-
def normalise(self, value):
|
|
204
|
-
# Some netcdf have pressue levels in float
|
|
205
|
-
if int(value) == value:
|
|
206
|
-
return int(value)
|
|
207
|
-
return value
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
class EnsembleCoordinate(Coordinate):
|
|
211
|
-
is_member = True
|
|
212
|
-
mars_names = ("number",)
|
|
213
|
-
|
|
214
|
-
def normalise(self, value):
|
|
215
|
-
if int(value) == value:
|
|
216
|
-
return int(value)
|
|
217
|
-
return value
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
class LongitudeCoordinate(Coordinate):
|
|
221
|
-
is_grid = True
|
|
222
|
-
is_lon = True
|
|
223
|
-
mars_names = ("longitude",)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
class LatitudeCoordinate(Coordinate):
|
|
227
|
-
is_grid = True
|
|
228
|
-
is_lat = True
|
|
229
|
-
mars_names = ("latitude",)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
class XCoordinate(Coordinate):
|
|
233
|
-
is_grid = True
|
|
234
|
-
is_x = True
|
|
235
|
-
mars_names = ("x",)
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
class YCoordinate(Coordinate):
|
|
239
|
-
is_grid = True
|
|
240
|
-
is_y = True
|
|
241
|
-
mars_names = ("y",)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
class ScalarCoordinate(Coordinate):
|
|
245
|
-
is_grid = False
|
|
246
|
-
|
|
247
|
-
@property
|
|
248
|
-
def mars_names(self):
|
|
249
|
-
return (self.variable.name,)
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
class UnsupportedCoordinate(Coordinate):
|
|
253
|
-
@property
|
|
254
|
-
def mars_names(self):
|
|
255
|
-
return (self.variable.name,)
|