anemoi-datasets 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anemoi/datasets/__init__.py +11 -3
- anemoi/datasets/__main__.py +2 -3
- anemoi/datasets/_version.py +2 -2
- anemoi/datasets/commands/__init__.py +2 -3
- anemoi/datasets/commands/cleanup.py +9 -0
- anemoi/datasets/commands/compare.py +3 -3
- anemoi/datasets/commands/copy.py +38 -68
- anemoi/datasets/commands/create.py +20 -5
- anemoi/datasets/commands/finalise-additions.py +9 -0
- anemoi/datasets/commands/finalise.py +9 -0
- anemoi/datasets/commands/init-additions.py +9 -0
- anemoi/datasets/commands/init.py +9 -0
- anemoi/datasets/commands/inspect.py +7 -1
- anemoi/datasets/commands/load-additions.py +9 -0
- anemoi/datasets/commands/load.py +9 -0
- anemoi/datasets/commands/patch.py +9 -0
- anemoi/datasets/commands/publish.py +9 -0
- anemoi/datasets/commands/scan.py +9 -0
- anemoi/datasets/compute/__init__.py +8 -0
- anemoi/datasets/compute/recentre.py +3 -2
- anemoi/datasets/create/__init__.py +64 -48
- anemoi/datasets/create/check.py +4 -3
- anemoi/datasets/create/chunks.py +3 -2
- anemoi/datasets/create/config.py +5 -5
- anemoi/datasets/create/functions/__init__.py +22 -7
- anemoi/datasets/create/functions/filters/__init__.py +2 -1
- anemoi/datasets/create/functions/filters/empty.py +3 -2
- anemoi/datasets/create/functions/filters/noop.py +2 -2
- anemoi/datasets/create/functions/filters/pressure_level_relative_humidity_to_specific_humidity.py +3 -2
- anemoi/datasets/create/functions/filters/pressure_level_specific_humidity_to_relative_humidity.py +3 -2
- anemoi/datasets/create/functions/filters/rename.py +16 -10
- anemoi/datasets/create/functions/filters/rotate_winds.py +3 -2
- anemoi/datasets/create/functions/filters/single_level_dewpoint_to_relative_humidity.py +3 -2
- anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_dewpoint.py +3 -2
- anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_specific_humidity.py +2 -2
- anemoi/datasets/create/functions/filters/single_level_specific_humidity_to_relative_humidity.py +2 -2
- anemoi/datasets/create/functions/filters/speeddir_to_uv.py +3 -2
- anemoi/datasets/create/functions/filters/unrotate_winds.py +3 -2
- anemoi/datasets/create/functions/filters/uv_to_speeddir.py +3 -2
- anemoi/datasets/create/functions/sources/__init__.py +2 -2
- anemoi/datasets/create/functions/sources/accumulations.py +10 -4
- anemoi/datasets/create/functions/sources/constants.py +3 -2
- anemoi/datasets/create/functions/sources/empty.py +3 -2
- anemoi/datasets/create/functions/sources/forcings.py +3 -2
- anemoi/datasets/create/functions/sources/grib.py +2 -2
- anemoi/datasets/create/functions/sources/hindcasts.py +3 -2
- anemoi/datasets/create/functions/sources/mars.py +97 -17
- anemoi/datasets/create/functions/sources/netcdf.py +3 -2
- anemoi/datasets/create/functions/sources/opendap.py +2 -2
- anemoi/datasets/create/functions/sources/recentre.py +3 -2
- anemoi/datasets/create/functions/sources/source.py +3 -2
- anemoi/datasets/create/functions/sources/tendencies.py +3 -2
- anemoi/datasets/create/functions/sources/xarray/__init__.py +8 -2
- anemoi/datasets/create/functions/sources/xarray/coordinates.py +5 -2
- anemoi/datasets/create/functions/sources/xarray/field.py +3 -2
- anemoi/datasets/create/functions/sources/xarray/fieldlist.py +12 -2
- anemoi/datasets/create/functions/sources/xarray/flavour.py +21 -16
- anemoi/datasets/create/functions/sources/xarray/grid.py +3 -2
- anemoi/datasets/create/functions/sources/xarray/metadata.py +3 -2
- anemoi/datasets/create/functions/sources/xarray/time.py +39 -4
- anemoi/datasets/create/functions/sources/xarray/variable.py +6 -6
- anemoi/datasets/create/functions/sources/xarray_kerchunk.py +2 -2
- anemoi/datasets/create/functions/sources/xarray_zarr.py +2 -2
- anemoi/datasets/create/functions/sources/zenodo.py +2 -2
- anemoi/datasets/create/input/__init__.py +3 -17
- anemoi/datasets/create/input/action.py +3 -2
- anemoi/datasets/create/input/concat.py +3 -2
- anemoi/datasets/create/input/context.py +3 -2
- anemoi/datasets/create/input/data_sources.py +3 -2
- anemoi/datasets/create/input/empty.py +3 -2
- anemoi/datasets/create/input/filter.py +3 -2
- anemoi/datasets/create/input/function.py +3 -2
- anemoi/datasets/create/input/join.py +3 -2
- anemoi/datasets/create/input/misc.py +3 -2
- anemoi/datasets/create/input/pipe.py +3 -2
- anemoi/datasets/create/input/repeated_dates.py +3 -2
- anemoi/datasets/create/input/result.py +187 -3
- anemoi/datasets/create/input/step.py +4 -2
- anemoi/datasets/create/input/template.py +3 -2
- anemoi/datasets/create/input/trace.py +3 -2
- anemoi/datasets/create/patch.py +9 -1
- anemoi/datasets/create/persistent.py +7 -3
- anemoi/datasets/create/size.py +3 -2
- anemoi/datasets/create/statistics/__init__.py +7 -3
- anemoi/datasets/create/statistics/summary.py +3 -2
- anemoi/datasets/create/utils.py +15 -2
- anemoi/datasets/create/writer.py +3 -2
- anemoi/datasets/create/zarr.py +8 -3
- anemoi/datasets/data/__init__.py +27 -1
- anemoi/datasets/data/concat.py +5 -1
- anemoi/datasets/data/dataset.py +216 -37
- anemoi/datasets/data/debug.py +4 -1
- anemoi/datasets/data/ensemble.py +4 -1
- anemoi/datasets/data/fill_missing.py +165 -0
- anemoi/datasets/data/forwards.py +27 -2
- anemoi/datasets/data/grids.py +236 -58
- anemoi/datasets/data/indexing.py +4 -1
- anemoi/datasets/data/interpolate.py +4 -1
- anemoi/datasets/data/join.py +17 -1
- anemoi/datasets/data/masked.py +36 -10
- anemoi/datasets/data/merge.py +180 -0
- anemoi/datasets/data/misc.py +18 -3
- anemoi/datasets/data/missing.py +4 -1
- anemoi/datasets/data/rescale.py +4 -1
- anemoi/datasets/data/select.py +15 -1
- anemoi/datasets/data/statistics.py +4 -1
- anemoi/datasets/data/stores.py +70 -3
- anemoi/datasets/data/subset.py +6 -1
- anemoi/datasets/data/unchecked.py +9 -1
- anemoi/datasets/data/xy.py +20 -5
- anemoi/datasets/dates/__init__.py +9 -7
- anemoi/datasets/dates/groups.py +3 -1
- anemoi/datasets/fields.py +3 -1
- anemoi/datasets/grids.py +86 -2
- anemoi/datasets/testing.py +60 -0
- anemoi/datasets/utils/__init__.py +8 -0
- anemoi/datasets/utils/fields.py +2 -2
- {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/METADATA +11 -29
- anemoi_datasets-0.5.10.dist-info/RECORD +124 -0
- {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/WHEEL +1 -1
- anemoi_datasets-0.5.6.dist-info/RECORD +0 -121
- {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/LICENSE +0 -0
- {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/entry_points.txt +0 -0
- {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/top_level.txt +0 -0
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
import datetime
|
|
11
|
+
import re
|
|
10
12
|
|
|
11
13
|
from anemoi.utils.humanize import did_you_mean
|
|
12
14
|
from earthkit.data import from_source
|
|
@@ -31,6 +33,25 @@ def _date_to_datetime(d):
|
|
|
31
33
|
return datetime.datetime.fromisoformat(d)
|
|
32
34
|
|
|
33
35
|
|
|
36
|
+
def expand_to_by(x):
|
|
37
|
+
|
|
38
|
+
if isinstance(x, (str, int)):
|
|
39
|
+
return expand_to_by(str(x).split("/"))
|
|
40
|
+
|
|
41
|
+
if len(x) == 3 and x[1] == "to":
|
|
42
|
+
start = int(x[0])
|
|
43
|
+
end = int(x[2])
|
|
44
|
+
return list(range(start, end + 1))
|
|
45
|
+
|
|
46
|
+
if len(x) == 5 and x[1] == "to" and x[3] == "by":
|
|
47
|
+
start = int(x[0])
|
|
48
|
+
end = int(x[2])
|
|
49
|
+
by = int(x[4])
|
|
50
|
+
return list(range(start, end + 1, by))
|
|
51
|
+
|
|
52
|
+
return x
|
|
53
|
+
|
|
54
|
+
|
|
34
55
|
def normalise_time_delta(t):
|
|
35
56
|
if isinstance(t, datetime.timedelta):
|
|
36
57
|
assert t == datetime.timedelta(hours=t.hours), t
|
|
@@ -42,25 +63,48 @@ def normalise_time_delta(t):
|
|
|
42
63
|
return t
|
|
43
64
|
|
|
44
65
|
|
|
66
|
+
def _normalise_time(t):
|
|
67
|
+
t = int(t)
|
|
68
|
+
if t < 100:
|
|
69
|
+
t * 100
|
|
70
|
+
return "{:04d}".format(t)
|
|
71
|
+
|
|
72
|
+
|
|
45
73
|
def _expand_mars_request(request, date, request_already_using_valid_datetime=False, date_key="date"):
|
|
46
74
|
requests = []
|
|
47
|
-
|
|
48
|
-
|
|
75
|
+
|
|
76
|
+
user_step = to_list(expand_to_by(request.get("step", [0])))
|
|
77
|
+
user_time = None
|
|
78
|
+
user_date = None
|
|
79
|
+
|
|
80
|
+
if not request_already_using_valid_datetime:
|
|
81
|
+
user_time = request.get("time")
|
|
82
|
+
if user_time is not None:
|
|
83
|
+
user_time = to_list(user_time)
|
|
84
|
+
user_time = [_normalise_time(t) for t in user_time]
|
|
85
|
+
|
|
86
|
+
user_date = request.get(date_key)
|
|
87
|
+
if user_date is not None:
|
|
88
|
+
assert isinstance(user_date, str), user_date
|
|
89
|
+
user_date = re.compile("^{}$".format(user_date.replace("-", "").replace("?", ".")))
|
|
90
|
+
|
|
91
|
+
for step in user_step:
|
|
49
92
|
r = request.copy()
|
|
50
93
|
|
|
51
94
|
if not request_already_using_valid_datetime:
|
|
52
95
|
|
|
53
|
-
if isinstance(
|
|
54
|
-
assert
|
|
96
|
+
if isinstance(step, str) and "-" in step:
|
|
97
|
+
assert step.count("-") == 1, step
|
|
98
|
+
|
|
55
99
|
# this takes care of the cases where the step is a period such as 0-24 or 12-24
|
|
56
|
-
hours = int(str(
|
|
100
|
+
hours = int(str(step).split("-")[-1])
|
|
57
101
|
|
|
58
102
|
base = date - datetime.timedelta(hours=hours)
|
|
59
103
|
r.update(
|
|
60
104
|
{
|
|
61
105
|
date_key: base.strftime("%Y%m%d"),
|
|
62
106
|
"time": base.strftime("%H%M"),
|
|
63
|
-
"step":
|
|
107
|
+
"step": step,
|
|
64
108
|
}
|
|
65
109
|
)
|
|
66
110
|
|
|
@@ -69,12 +113,28 @@ def _expand_mars_request(request, date, request_already_using_valid_datetime=Fal
|
|
|
69
113
|
if isinstance(r[pproc], (list, tuple)):
|
|
70
114
|
r[pproc] = "/".join(str(x) for x in r[pproc])
|
|
71
115
|
|
|
116
|
+
if user_date is not None:
|
|
117
|
+
if not user_date.match(r[date_key]):
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
if user_time is not None:
|
|
121
|
+
# It time is provided by the user, we only keep the requests that match the time
|
|
122
|
+
if r["time"] not in user_time:
|
|
123
|
+
continue
|
|
124
|
+
|
|
72
125
|
requests.append(r)
|
|
73
126
|
|
|
127
|
+
# assert requests, requests
|
|
128
|
+
|
|
74
129
|
return requests
|
|
75
130
|
|
|
76
131
|
|
|
77
|
-
def factorise_requests(
|
|
132
|
+
def factorise_requests(
|
|
133
|
+
dates,
|
|
134
|
+
*requests,
|
|
135
|
+
request_already_using_valid_datetime=False,
|
|
136
|
+
date_key="date",
|
|
137
|
+
):
|
|
78
138
|
updates = []
|
|
79
139
|
for req in requests:
|
|
80
140
|
# req = normalise_request(req)
|
|
@@ -87,6 +147,9 @@ def factorise_requests(dates, *requests, request_already_using_valid_datetime=Fa
|
|
|
87
147
|
date_key=date_key,
|
|
88
148
|
)
|
|
89
149
|
|
|
150
|
+
if not updates:
|
|
151
|
+
return
|
|
152
|
+
|
|
90
153
|
compressed = Availability(updates)
|
|
91
154
|
for r in compressed.iterate():
|
|
92
155
|
for k, v in r.items():
|
|
@@ -177,13 +240,24 @@ MARS_KEYS = [
|
|
|
177
240
|
]
|
|
178
241
|
|
|
179
242
|
|
|
180
|
-
def mars(
|
|
243
|
+
def mars(
|
|
244
|
+
context,
|
|
245
|
+
dates,
|
|
246
|
+
*requests,
|
|
247
|
+
request_already_using_valid_datetime=False,
|
|
248
|
+
date_key="date",
|
|
249
|
+
**kwargs,
|
|
250
|
+
):
|
|
251
|
+
|
|
181
252
|
if not requests:
|
|
182
253
|
requests = [kwargs]
|
|
183
254
|
|
|
184
255
|
for r in requests:
|
|
256
|
+
param = r.get("param", [])
|
|
257
|
+
if not isinstance(param, (list, tuple)):
|
|
258
|
+
param = [param]
|
|
185
259
|
# check for "Norway bug" where yaml transforms 'no' into False, etc.
|
|
186
|
-
for p in
|
|
260
|
+
for p in param:
|
|
187
261
|
if p is False:
|
|
188
262
|
raise ValueError(
|
|
189
263
|
"'param' cannot be 'False'. If you wrote 'param: no' or 'param: off' in yaml, you may want to use quotes?"
|
|
@@ -197,12 +271,18 @@ def mars(context, dates, *requests, request_already_using_valid_datetime=False,
|
|
|
197
271
|
"'param' cannot be 'True'. If you wrote 'param: on' in yaml, you may want to use quotes?"
|
|
198
272
|
)
|
|
199
273
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
274
|
+
if len(dates) == 0: # When using `repeated_dates`
|
|
275
|
+
assert len(requests) == 1, requests
|
|
276
|
+
assert "date" in requests[0], requests[0]
|
|
277
|
+
if isinstance(requests[0]["date"], datetime.date):
|
|
278
|
+
requests[0]["date"] = requests[0]["date"].strftime("%Y%m%d")
|
|
279
|
+
else:
|
|
280
|
+
requests = factorise_requests(
|
|
281
|
+
dates,
|
|
282
|
+
*requests,
|
|
283
|
+
request_already_using_valid_datetime=request_already_using_valid_datetime,
|
|
284
|
+
date_key=date_key,
|
|
285
|
+
)
|
|
206
286
|
|
|
207
287
|
requests = list(requests)
|
|
208
288
|
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
from .xarray import load_many
|
|
11
12
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
from .xarray import load_many
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
from copy import deepcopy
|
|
10
11
|
|
|
11
12
|
from anemoi.datasets.compute.recentre import recentre as _recentre
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
from earthkit.data import from_source
|
|
10
11
|
|
|
11
12
|
from anemoi.datasets.create.utils import to_datetime_list
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
import datetime
|
|
10
11
|
from collections import defaultdict
|
|
11
12
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
11
|
|
|
@@ -46,6 +46,12 @@ def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs)
|
|
|
46
46
|
|
|
47
47
|
if isinstance(dataset, str) and ".zarr" in dataset:
|
|
48
48
|
data = xr.open_zarr(name_to_zarr_store(dataset), **options)
|
|
49
|
+
elif "planetarycomputer" in dataset:
|
|
50
|
+
store = name_to_zarr_store(dataset)
|
|
51
|
+
if "store" in store:
|
|
52
|
+
data = xr.open_zarr(**store)
|
|
53
|
+
if "filename_or_obj" in store:
|
|
54
|
+
data = xr.open_dataset(**store)
|
|
49
55
|
else:
|
|
50
56
|
data = xr.open_dataset(dataset, **options)
|
|
51
57
|
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
import datetime
|
|
11
12
|
import logging
|
|
@@ -30,6 +31,8 @@ def extract_single_value(variable):
|
|
|
30
31
|
if np.issubdtype(variable.values.dtype, np.datetime64):
|
|
31
32
|
if len(shape) == 0:
|
|
32
33
|
return to_datetime(variable.values) # Convert to python datetime
|
|
34
|
+
if shape == (1,):
|
|
35
|
+
return to_datetime(variable.values[0])
|
|
33
36
|
assert False, (shape, variable.values[:2])
|
|
34
37
|
|
|
35
38
|
if np.issubdtype(variable.values.dtype, np.timedelta64):
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
import datetime
|
|
11
12
|
import logging
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
import json
|
|
11
12
|
import logging
|
|
@@ -60,6 +61,15 @@ class XarrayFieldList(FieldList):
|
|
|
60
61
|
else:
|
|
61
62
|
flavour = json.load(f)
|
|
62
63
|
|
|
64
|
+
if isinstance(flavour, dict):
|
|
65
|
+
flavour_coords = [coords["name"] for coords in flavour["rules"].values()]
|
|
66
|
+
ds_dims = [dim for dim in ds._dims]
|
|
67
|
+
for dim in ds_dims:
|
|
68
|
+
if dim in flavour_coords and dim not in ds._coord_names:
|
|
69
|
+
ds = ds.assign_coords({dim: ds[dim]})
|
|
70
|
+
else:
|
|
71
|
+
pass
|
|
72
|
+
|
|
63
73
|
guess = CoordinateGuesser.from_flavour(ds, flavour)
|
|
64
74
|
|
|
65
75
|
skip = set()
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
import logging
|
|
@@ -228,20 +228,15 @@ class CoordinateGuesser:
|
|
|
228
228
|
x = x[0]
|
|
229
229
|
y = y[0]
|
|
230
230
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
if x.variable.dims != y.variable.dims:
|
|
234
|
-
raise ValueError(f"Dimensions do not match {x.name}{x.variable.dims} != {y.name}{y.variable.dims}")
|
|
235
|
-
|
|
236
|
-
if (x.name, y.name) in self._cache:
|
|
237
|
-
return self._cache[(x.name, y.name)]
|
|
238
|
-
|
|
239
|
-
if (x.name, y.name) in self._cache:
|
|
240
|
-
return self._cache[(x.name, y.name)]
|
|
231
|
+
dim_vars, unstructured = self._check_dims(variable, x, y)
|
|
241
232
|
|
|
242
|
-
|
|
233
|
+
if (x.name, y.name, dim_vars) in self._cache:
|
|
234
|
+
return self._cache[(x.name, y.name, dim_vars)]
|
|
243
235
|
|
|
244
236
|
grid_mapping = variable.attrs.get("grid_mapping", None)
|
|
237
|
+
if grid_mapping is not None:
|
|
238
|
+
print(f"grid_mapping: {grid_mapping}")
|
|
239
|
+
print(self.ds[grid_mapping])
|
|
245
240
|
|
|
246
241
|
if grid_mapping is None:
|
|
247
242
|
LOG.warning(f"No 'grid_mapping' attribute provided for '{variable.name}'")
|
|
@@ -288,11 +283,19 @@ class CoordinateGuesser:
|
|
|
288
283
|
grid_mapping = self.ds.attrs["crs"]
|
|
289
284
|
LOG.warning(f"Using CRS {grid_mapping} from global attributes")
|
|
290
285
|
|
|
286
|
+
grid = None
|
|
291
287
|
if grid_mapping is not None:
|
|
288
|
+
|
|
289
|
+
grid_mapping = dict(self.ds[grid_mapping].attrs)
|
|
290
|
+
|
|
292
291
|
if unstructured:
|
|
293
|
-
|
|
292
|
+
grid = UnstructuredProjectionGrid(x, y, grid_mapping)
|
|
294
293
|
else:
|
|
295
|
-
|
|
294
|
+
grid = MeshProjectionGrid(x, y, grid_mapping)
|
|
295
|
+
|
|
296
|
+
if grid is not None:
|
|
297
|
+
self._cache[(x.name, y.name, dim_vars)] = grid
|
|
298
|
+
return grid
|
|
296
299
|
|
|
297
300
|
LOG.error("Could not fine a candidate for 'grid_mapping'")
|
|
298
301
|
raise NotImplementedError(f"Unstructured grid {x.name} {y.name}")
|
|
@@ -340,12 +343,14 @@ class DefaultCoordinateGuesser(CoordinateGuesser):
|
|
|
340
343
|
if standard_name == "time":
|
|
341
344
|
return TimeCoordinate(c)
|
|
342
345
|
|
|
343
|
-
|
|
346
|
+
# That is the output of `cfgrib` for forecasts
|
|
347
|
+
if name == "time" and standard_name != "forecast_reference_time":
|
|
344
348
|
return TimeCoordinate(c)
|
|
345
349
|
|
|
346
350
|
def _is_date(self, c, *, axis, name, long_name, standard_name, units):
|
|
347
351
|
if standard_name == "forecast_reference_time":
|
|
348
352
|
return DateCoordinate(c)
|
|
353
|
+
|
|
349
354
|
if name == "forecast_reference_time":
|
|
350
355
|
return DateCoordinate(c)
|
|
351
356
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
import logging
|
|
@@ -100,6 +100,7 @@ class ProjectionGrid(XYGrid):
|
|
|
100
100
|
data_crs = CRS.from_cf(self.projection)
|
|
101
101
|
else:
|
|
102
102
|
data_crs = self.projection
|
|
103
|
+
|
|
103
104
|
wgs84_crs = CRS.from_epsg(4326) # WGS84
|
|
104
105
|
|
|
105
106
|
return Transformer.from_crs(data_crs, wgs84_crs, always_xy=True)
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
import logging
|
|
11
12
|
from functools import cached_property
|
|
@@ -1,17 +1,20 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
import datetime
|
|
12
|
+
import logging
|
|
12
13
|
|
|
13
14
|
from anemoi.utils.dates import as_datetime
|
|
14
15
|
|
|
16
|
+
LOG = logging.getLogger(__name__)
|
|
17
|
+
|
|
15
18
|
|
|
16
19
|
class Time:
|
|
17
20
|
|
|
@@ -36,7 +39,28 @@ class Time:
|
|
|
36
39
|
if len(date_coordinate) == 1 and len(time_coordinate) == 0 and len(step_coordinate) == 1:
|
|
37
40
|
return ForecastFromBaseTimeAndDate(date_coordinate[0], step_coordinate[0])
|
|
38
41
|
|
|
39
|
-
|
|
42
|
+
if len(date_coordinate) == 1 and len(time_coordinate) == 1 and len(step_coordinate) == 1:
|
|
43
|
+
return ForecastFromValidTimeAndStep(time_coordinate[0], step_coordinate[0], date_coordinate[0])
|
|
44
|
+
|
|
45
|
+
LOG.error("")
|
|
46
|
+
LOG.error(f"{len(date_coordinate)} date_coordinate")
|
|
47
|
+
for c in date_coordinate:
|
|
48
|
+
LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
|
|
49
|
+
# LOG.error(' %s', c.variable)
|
|
50
|
+
|
|
51
|
+
LOG.error("")
|
|
52
|
+
LOG.error(f"{len(time_coordinate)} time_coordinate")
|
|
53
|
+
for c in time_coordinate:
|
|
54
|
+
LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
|
|
55
|
+
# LOG.error(' %s', c.variable)
|
|
56
|
+
|
|
57
|
+
LOG.error("")
|
|
58
|
+
LOG.error(f"{len(step_coordinate)} step_coordinate")
|
|
59
|
+
for c in step_coordinate:
|
|
60
|
+
LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
|
|
61
|
+
# LOG.error(' %s', c.variable)
|
|
62
|
+
|
|
63
|
+
raise NotImplementedError(f"{len(date_coordinate)=} {len(time_coordinate)=} {len(step_coordinate)=}")
|
|
40
64
|
|
|
41
65
|
|
|
42
66
|
class Constant(Time):
|
|
@@ -62,9 +86,10 @@ class Analysis(Time):
|
|
|
62
86
|
|
|
63
87
|
class ForecastFromValidTimeAndStep(Time):
|
|
64
88
|
|
|
65
|
-
def __init__(self, time_coordinate, step_coordinate):
|
|
89
|
+
def __init__(self, time_coordinate, step_coordinate, date_coordinate=None):
|
|
66
90
|
self.time_coordinate_name = time_coordinate.variable.name
|
|
67
91
|
self.step_coordinate_name = step_coordinate.variable.name
|
|
92
|
+
self.date_coordinate_name = date_coordinate.variable.name if date_coordinate else None
|
|
68
93
|
|
|
69
94
|
def fill_time_metadata(self, coords_values, metadata):
|
|
70
95
|
valid_datetime = coords_values[self.time_coordinate_name]
|
|
@@ -79,6 +104,16 @@ class ForecastFromValidTimeAndStep(Time):
|
|
|
79
104
|
metadata["date"] = as_datetime(base_datetime).strftime("%Y%m%d")
|
|
80
105
|
metadata["time"] = as_datetime(base_datetime).strftime("%H%M")
|
|
81
106
|
metadata["step"] = int(hours)
|
|
107
|
+
|
|
108
|
+
# When date is present, it should be compatible with time and step
|
|
109
|
+
|
|
110
|
+
if self.date_coordinate_name is not None:
|
|
111
|
+
# Not sure that this is the correct assumption
|
|
112
|
+
assert coords_values[self.date_coordinate_name] == base_datetime, (
|
|
113
|
+
coords_values[self.date_coordinate_name],
|
|
114
|
+
base_datetime,
|
|
115
|
+
)
|
|
116
|
+
|
|
82
117
|
return valid_datetime
|
|
83
118
|
|
|
84
119
|
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
+
|
|
9
10
|
|
|
10
11
|
import logging
|
|
11
12
|
import math
|
|
@@ -82,9 +83,8 @@ class Variable:
|
|
|
82
83
|
)
|
|
83
84
|
|
|
84
85
|
def __getitem__(self, i):
|
|
85
|
-
"""
|
|
86
|
-
|
|
87
|
-
"""
|
|
86
|
+
"""Get a 2D field from the variable"""
|
|
87
|
+
|
|
88
88
|
if i >= self.length:
|
|
89
89
|
raise IndexError(i)
|
|
90
90
|
|
|
@@ -117,7 +117,7 @@ class Variable:
|
|
|
117
117
|
|
|
118
118
|
variable = Variable(
|
|
119
119
|
ds=self.ds,
|
|
120
|
-
|
|
120
|
+
variable=self.variable.isel({k: i}),
|
|
121
121
|
coordinates=coordinates,
|
|
122
122
|
grid=self.grid,
|
|
123
123
|
time=self.time,
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
from earthkit.data.core.fieldlist import MultiFieldList
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
from .xarray import load_many
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# (C) Copyright 2024
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
#
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
from earthkit.data.core.fieldlist import MultiFieldList
|
|
@@ -1,28 +1,14 @@
|
|
|
1
|
-
# (C) Copyright
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
2
|
#
|
|
3
3
|
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
4
|
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
5
6
|
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
6
7
|
# granted to it by virtue of its status as an intergovernmental organisation
|
|
7
8
|
# nor does it submit to any jurisdiction.
|
|
8
|
-
|
|
9
|
-
import datetime
|
|
10
|
-
import itertools
|
|
9
|
+
|
|
11
10
|
import logging
|
|
12
|
-
import math
|
|
13
|
-
import time
|
|
14
|
-
from collections import defaultdict
|
|
15
11
|
from copy import deepcopy
|
|
16
|
-
from functools import cached_property
|
|
17
|
-
from functools import wraps
|
|
18
|
-
|
|
19
|
-
import numpy as np
|
|
20
|
-
from anemoi.utils.dates import as_datetime as as_datetime
|
|
21
|
-
from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
|
|
22
|
-
|
|
23
|
-
from anemoi.datasets.dates import DatesProvider as DatesProvider
|
|
24
|
-
from anemoi.datasets.fields import FieldArray as FieldArray
|
|
25
|
-
from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
|
|
26
12
|
|
|
27
13
|
from .trace import trace_select
|
|
28
14
|
|