anemoi-datasets 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. anemoi/datasets/__init__.py +11 -3
  2. anemoi/datasets/__main__.py +2 -3
  3. anemoi/datasets/_version.py +2 -2
  4. anemoi/datasets/commands/__init__.py +2 -3
  5. anemoi/datasets/commands/cleanup.py +9 -0
  6. anemoi/datasets/commands/compare.py +3 -3
  7. anemoi/datasets/commands/copy.py +38 -68
  8. anemoi/datasets/commands/create.py +20 -5
  9. anemoi/datasets/commands/finalise-additions.py +9 -0
  10. anemoi/datasets/commands/finalise.py +9 -0
  11. anemoi/datasets/commands/init-additions.py +9 -0
  12. anemoi/datasets/commands/init.py +9 -0
  13. anemoi/datasets/commands/inspect.py +7 -1
  14. anemoi/datasets/commands/load-additions.py +9 -0
  15. anemoi/datasets/commands/load.py +9 -0
  16. anemoi/datasets/commands/patch.py +9 -0
  17. anemoi/datasets/commands/publish.py +9 -0
  18. anemoi/datasets/commands/scan.py +9 -0
  19. anemoi/datasets/compute/__init__.py +8 -0
  20. anemoi/datasets/compute/recentre.py +3 -2
  21. anemoi/datasets/create/__init__.py +64 -48
  22. anemoi/datasets/create/check.py +4 -3
  23. anemoi/datasets/create/chunks.py +3 -2
  24. anemoi/datasets/create/config.py +5 -5
  25. anemoi/datasets/create/functions/__init__.py +22 -7
  26. anemoi/datasets/create/functions/filters/__init__.py +2 -1
  27. anemoi/datasets/create/functions/filters/empty.py +3 -2
  28. anemoi/datasets/create/functions/filters/noop.py +2 -2
  29. anemoi/datasets/create/functions/filters/pressure_level_relative_humidity_to_specific_humidity.py +3 -2
  30. anemoi/datasets/create/functions/filters/pressure_level_specific_humidity_to_relative_humidity.py +3 -2
  31. anemoi/datasets/create/functions/filters/rename.py +16 -10
  32. anemoi/datasets/create/functions/filters/rotate_winds.py +3 -2
  33. anemoi/datasets/create/functions/filters/single_level_dewpoint_to_relative_humidity.py +3 -2
  34. anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_dewpoint.py +3 -2
  35. anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_specific_humidity.py +2 -2
  36. anemoi/datasets/create/functions/filters/single_level_specific_humidity_to_relative_humidity.py +2 -2
  37. anemoi/datasets/create/functions/filters/speeddir_to_uv.py +3 -2
  38. anemoi/datasets/create/functions/filters/unrotate_winds.py +3 -2
  39. anemoi/datasets/create/functions/filters/uv_to_speeddir.py +3 -2
  40. anemoi/datasets/create/functions/sources/__init__.py +2 -2
  41. anemoi/datasets/create/functions/sources/accumulations.py +10 -4
  42. anemoi/datasets/create/functions/sources/constants.py +3 -2
  43. anemoi/datasets/create/functions/sources/empty.py +3 -2
  44. anemoi/datasets/create/functions/sources/forcings.py +3 -2
  45. anemoi/datasets/create/functions/sources/grib.py +2 -2
  46. anemoi/datasets/create/functions/sources/hindcasts.py +3 -2
  47. anemoi/datasets/create/functions/sources/mars.py +97 -17
  48. anemoi/datasets/create/functions/sources/netcdf.py +3 -2
  49. anemoi/datasets/create/functions/sources/opendap.py +2 -2
  50. anemoi/datasets/create/functions/sources/recentre.py +3 -2
  51. anemoi/datasets/create/functions/sources/source.py +3 -2
  52. anemoi/datasets/create/functions/sources/tendencies.py +3 -2
  53. anemoi/datasets/create/functions/sources/xarray/__init__.py +8 -2
  54. anemoi/datasets/create/functions/sources/xarray/coordinates.py +5 -2
  55. anemoi/datasets/create/functions/sources/xarray/field.py +3 -2
  56. anemoi/datasets/create/functions/sources/xarray/fieldlist.py +12 -2
  57. anemoi/datasets/create/functions/sources/xarray/flavour.py +21 -16
  58. anemoi/datasets/create/functions/sources/xarray/grid.py +3 -2
  59. anemoi/datasets/create/functions/sources/xarray/metadata.py +3 -2
  60. anemoi/datasets/create/functions/sources/xarray/time.py +39 -4
  61. anemoi/datasets/create/functions/sources/xarray/variable.py +6 -6
  62. anemoi/datasets/create/functions/sources/xarray_kerchunk.py +2 -2
  63. anemoi/datasets/create/functions/sources/xarray_zarr.py +2 -2
  64. anemoi/datasets/create/functions/sources/zenodo.py +2 -2
  65. anemoi/datasets/create/input/__init__.py +3 -17
  66. anemoi/datasets/create/input/action.py +3 -2
  67. anemoi/datasets/create/input/concat.py +3 -2
  68. anemoi/datasets/create/input/context.py +3 -2
  69. anemoi/datasets/create/input/data_sources.py +3 -2
  70. anemoi/datasets/create/input/empty.py +3 -2
  71. anemoi/datasets/create/input/filter.py +3 -2
  72. anemoi/datasets/create/input/function.py +3 -2
  73. anemoi/datasets/create/input/join.py +3 -2
  74. anemoi/datasets/create/input/misc.py +3 -2
  75. anemoi/datasets/create/input/pipe.py +3 -2
  76. anemoi/datasets/create/input/repeated_dates.py +3 -2
  77. anemoi/datasets/create/input/result.py +187 -3
  78. anemoi/datasets/create/input/step.py +4 -2
  79. anemoi/datasets/create/input/template.py +3 -2
  80. anemoi/datasets/create/input/trace.py +3 -2
  81. anemoi/datasets/create/patch.py +9 -1
  82. anemoi/datasets/create/persistent.py +7 -3
  83. anemoi/datasets/create/size.py +3 -2
  84. anemoi/datasets/create/statistics/__init__.py +7 -3
  85. anemoi/datasets/create/statistics/summary.py +3 -2
  86. anemoi/datasets/create/utils.py +15 -2
  87. anemoi/datasets/create/writer.py +3 -2
  88. anemoi/datasets/create/zarr.py +8 -3
  89. anemoi/datasets/data/__init__.py +27 -1
  90. anemoi/datasets/data/concat.py +5 -1
  91. anemoi/datasets/data/dataset.py +216 -37
  92. anemoi/datasets/data/debug.py +4 -1
  93. anemoi/datasets/data/ensemble.py +4 -1
  94. anemoi/datasets/data/fill_missing.py +165 -0
  95. anemoi/datasets/data/forwards.py +27 -2
  96. anemoi/datasets/data/grids.py +236 -58
  97. anemoi/datasets/data/indexing.py +4 -1
  98. anemoi/datasets/data/interpolate.py +4 -1
  99. anemoi/datasets/data/join.py +17 -1
  100. anemoi/datasets/data/masked.py +36 -10
  101. anemoi/datasets/data/merge.py +180 -0
  102. anemoi/datasets/data/misc.py +18 -3
  103. anemoi/datasets/data/missing.py +4 -1
  104. anemoi/datasets/data/rescale.py +4 -1
  105. anemoi/datasets/data/select.py +15 -1
  106. anemoi/datasets/data/statistics.py +4 -1
  107. anemoi/datasets/data/stores.py +70 -3
  108. anemoi/datasets/data/subset.py +6 -1
  109. anemoi/datasets/data/unchecked.py +9 -1
  110. anemoi/datasets/data/xy.py +20 -5
  111. anemoi/datasets/dates/__init__.py +9 -7
  112. anemoi/datasets/dates/groups.py +3 -1
  113. anemoi/datasets/fields.py +3 -1
  114. anemoi/datasets/grids.py +86 -2
  115. anemoi/datasets/testing.py +60 -0
  116. anemoi/datasets/utils/__init__.py +8 -0
  117. anemoi/datasets/utils/fields.py +2 -2
  118. {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/METADATA +11 -29
  119. anemoi_datasets-0.5.10.dist-info/RECORD +124 -0
  120. {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/WHEEL +1 -1
  121. anemoi_datasets-0.5.6.dist-info/RECORD +0 -121
  122. {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/LICENSE +0 -0
  123. {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/entry_points.txt +0 -0
  124. {anemoi_datasets-0.5.6.dist-info → anemoi_datasets-0.5.10.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,14 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
  import datetime
11
+ import re
10
12
 
11
13
  from anemoi.utils.humanize import did_you_mean
12
14
  from earthkit.data import from_source
@@ -31,6 +33,25 @@ def _date_to_datetime(d):
31
33
  return datetime.datetime.fromisoformat(d)
32
34
 
33
35
 
36
+ def expand_to_by(x):
37
+
38
+ if isinstance(x, (str, int)):
39
+ return expand_to_by(str(x).split("/"))
40
+
41
+ if len(x) == 3 and x[1] == "to":
42
+ start = int(x[0])
43
+ end = int(x[2])
44
+ return list(range(start, end + 1))
45
+
46
+ if len(x) == 5 and x[1] == "to" and x[3] == "by":
47
+ start = int(x[0])
48
+ end = int(x[2])
49
+ by = int(x[4])
50
+ return list(range(start, end + 1, by))
51
+
52
+ return x
53
+
54
+
34
55
  def normalise_time_delta(t):
35
56
  if isinstance(t, datetime.timedelta):
36
57
  assert t == datetime.timedelta(hours=t.hours), t
@@ -42,25 +63,48 @@ def normalise_time_delta(t):
42
63
  return t
43
64
 
44
65
 
66
+ def _normalise_time(t):
67
+ t = int(t)
68
+ if t < 100:
69
+ t * 100
70
+ return "{:04d}".format(t)
71
+
72
+
45
73
  def _expand_mars_request(request, date, request_already_using_valid_datetime=False, date_key="date"):
46
74
  requests = []
47
- step = to_list(request.get("step", [0]))
48
- for s in step:
75
+
76
+ user_step = to_list(expand_to_by(request.get("step", [0])))
77
+ user_time = None
78
+ user_date = None
79
+
80
+ if not request_already_using_valid_datetime:
81
+ user_time = request.get("time")
82
+ if user_time is not None:
83
+ user_time = to_list(user_time)
84
+ user_time = [_normalise_time(t) for t in user_time]
85
+
86
+ user_date = request.get(date_key)
87
+ if user_date is not None:
88
+ assert isinstance(user_date, str), user_date
89
+ user_date = re.compile("^{}$".format(user_date.replace("-", "").replace("?", ".")))
90
+
91
+ for step in user_step:
49
92
  r = request.copy()
50
93
 
51
94
  if not request_already_using_valid_datetime:
52
95
 
53
- if isinstance(s, str) and "-" in s:
54
- assert s.count("-") == 1, s
96
+ if isinstance(step, str) and "-" in step:
97
+ assert step.count("-") == 1, step
98
+
55
99
  # this takes care of the cases where the step is a period such as 0-24 or 12-24
56
- hours = int(str(s).split("-")[-1])
100
+ hours = int(str(step).split("-")[-1])
57
101
 
58
102
  base = date - datetime.timedelta(hours=hours)
59
103
  r.update(
60
104
  {
61
105
  date_key: base.strftime("%Y%m%d"),
62
106
  "time": base.strftime("%H%M"),
63
- "step": s,
107
+ "step": step,
64
108
  }
65
109
  )
66
110
 
@@ -69,12 +113,28 @@ def _expand_mars_request(request, date, request_already_using_valid_datetime=Fal
69
113
  if isinstance(r[pproc], (list, tuple)):
70
114
  r[pproc] = "/".join(str(x) for x in r[pproc])
71
115
 
116
+ if user_date is not None:
117
+ if not user_date.match(r[date_key]):
118
+ continue
119
+
120
+ if user_time is not None:
121
+ # It time is provided by the user, we only keep the requests that match the time
122
+ if r["time"] not in user_time:
123
+ continue
124
+
72
125
  requests.append(r)
73
126
 
127
+ # assert requests, requests
128
+
74
129
  return requests
75
130
 
76
131
 
77
- def factorise_requests(dates, *requests, request_already_using_valid_datetime=False, date_key="date"):
132
+ def factorise_requests(
133
+ dates,
134
+ *requests,
135
+ request_already_using_valid_datetime=False,
136
+ date_key="date",
137
+ ):
78
138
  updates = []
79
139
  for req in requests:
80
140
  # req = normalise_request(req)
@@ -87,6 +147,9 @@ def factorise_requests(dates, *requests, request_already_using_valid_datetime=Fa
87
147
  date_key=date_key,
88
148
  )
89
149
 
150
+ if not updates:
151
+ return
152
+
90
153
  compressed = Availability(updates)
91
154
  for r in compressed.iterate():
92
155
  for k, v in r.items():
@@ -177,13 +240,24 @@ MARS_KEYS = [
177
240
  ]
178
241
 
179
242
 
180
- def mars(context, dates, *requests, request_already_using_valid_datetime=False, date_key="date", **kwargs):
243
+ def mars(
244
+ context,
245
+ dates,
246
+ *requests,
247
+ request_already_using_valid_datetime=False,
248
+ date_key="date",
249
+ **kwargs,
250
+ ):
251
+
181
252
  if not requests:
182
253
  requests = [kwargs]
183
254
 
184
255
  for r in requests:
256
+ param = r.get("param", [])
257
+ if not isinstance(param, (list, tuple)):
258
+ param = [param]
185
259
  # check for "Norway bug" where yaml transforms 'no' into False, etc.
186
- for p in r.get("param", []):
260
+ for p in param:
187
261
  if p is False:
188
262
  raise ValueError(
189
263
  "'param' cannot be 'False'. If you wrote 'param: no' or 'param: off' in yaml, you may want to use quotes?"
@@ -197,12 +271,18 @@ def mars(context, dates, *requests, request_already_using_valid_datetime=False,
197
271
  "'param' cannot be 'True'. If you wrote 'param: on' in yaml, you may want to use quotes?"
198
272
  )
199
273
 
200
- requests = factorise_requests(
201
- dates,
202
- *requests,
203
- request_already_using_valid_datetime=request_already_using_valid_datetime,
204
- date_key=date_key,
205
- )
274
+ if len(dates) == 0: # When using `repeated_dates`
275
+ assert len(requests) == 1, requests
276
+ assert "date" in requests[0], requests[0]
277
+ if isinstance(requests[0]["date"], datetime.date):
278
+ requests[0]["date"] = requests[0]["date"].strftime("%Y%m%d")
279
+ else:
280
+ requests = factorise_requests(
281
+ dates,
282
+ *requests,
283
+ request_already_using_valid_datetime=request_already_using_valid_datetime,
284
+ date_key=date_key,
285
+ )
206
286
 
207
287
  requests = list(requests)
208
288
 
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  from .xarray import load_many
11
12
 
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  from .xarray import load_many
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
  from copy import deepcopy
10
11
 
11
12
  from anemoi.datasets.compute.recentre import recentre as _recentre
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
  from earthkit.data import from_source
10
11
 
11
12
  from anemoi.datasets.create.utils import to_datetime_list
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
  import datetime
10
11
  from collections import defaultdict
11
12
 
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
  import logging
11
11
 
@@ -46,6 +46,12 @@ def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs)
46
46
 
47
47
  if isinstance(dataset, str) and ".zarr" in dataset:
48
48
  data = xr.open_zarr(name_to_zarr_store(dataset), **options)
49
+ elif "planetarycomputer" in dataset:
50
+ store = name_to_zarr_store(dataset)
51
+ if "store" in store:
52
+ data = xr.open_zarr(**store)
53
+ if "filename_or_obj" in store:
54
+ data = xr.open_dataset(**store)
49
55
  else:
50
56
  data = xr.open_dataset(dataset, **options)
51
57
 
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  import datetime
11
12
  import logging
@@ -30,6 +31,8 @@ def extract_single_value(variable):
30
31
  if np.issubdtype(variable.values.dtype, np.datetime64):
31
32
  if len(shape) == 0:
32
33
  return to_datetime(variable.values) # Convert to python datetime
34
+ if shape == (1,):
35
+ return to_datetime(variable.values[0])
33
36
  assert False, (shape, variable.values[:2])
34
37
 
35
38
  if np.issubdtype(variable.values.dtype, np.timedelta64):
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  import datetime
11
12
  import logging
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  import json
11
12
  import logging
@@ -60,6 +61,15 @@ class XarrayFieldList(FieldList):
60
61
  else:
61
62
  flavour = json.load(f)
62
63
 
64
+ if isinstance(flavour, dict):
65
+ flavour_coords = [coords["name"] for coords in flavour["rules"].values()]
66
+ ds_dims = [dim for dim in ds._dims]
67
+ for dim in ds_dims:
68
+ if dim in flavour_coords and dim not in ds._coord_names:
69
+ ds = ds.assign_coords({dim: ds[dim]})
70
+ else:
71
+ pass
72
+
63
73
  guess = CoordinateGuesser.from_flavour(ds, flavour)
64
74
 
65
75
  skip = set()
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  import logging
@@ -228,20 +228,15 @@ class CoordinateGuesser:
228
228
  x = x[0]
229
229
  y = y[0]
230
230
 
231
- _, unstructured = self._check_dims(variable, x, y)
232
-
233
- if x.variable.dims != y.variable.dims:
234
- raise ValueError(f"Dimensions do not match {x.name}{x.variable.dims} != {y.name}{y.variable.dims}")
235
-
236
- if (x.name, y.name) in self._cache:
237
- return self._cache[(x.name, y.name)]
238
-
239
- if (x.name, y.name) in self._cache:
240
- return self._cache[(x.name, y.name)]
231
+ dim_vars, unstructured = self._check_dims(variable, x, y)
241
232
 
242
- assert len(x.variable.shape) == len(y.variable.shape), (x.variable.shape, y.variable.shape)
233
+ if (x.name, y.name, dim_vars) in self._cache:
234
+ return self._cache[(x.name, y.name, dim_vars)]
243
235
 
244
236
  grid_mapping = variable.attrs.get("grid_mapping", None)
237
+ if grid_mapping is not None:
238
+ print(f"grid_mapping: {grid_mapping}")
239
+ print(self.ds[grid_mapping])
245
240
 
246
241
  if grid_mapping is None:
247
242
  LOG.warning(f"No 'grid_mapping' attribute provided for '{variable.name}'")
@@ -288,11 +283,19 @@ class CoordinateGuesser:
288
283
  grid_mapping = self.ds.attrs["crs"]
289
284
  LOG.warning(f"Using CRS {grid_mapping} from global attributes")
290
285
 
286
+ grid = None
291
287
  if grid_mapping is not None:
288
+
289
+ grid_mapping = dict(self.ds[grid_mapping].attrs)
290
+
292
291
  if unstructured:
293
- return UnstructuredProjectionGrid(x, y, grid_mapping)
292
+ grid = UnstructuredProjectionGrid(x, y, grid_mapping)
294
293
  else:
295
- return MeshProjectionGrid(x, y, grid_mapping)
294
+ grid = MeshProjectionGrid(x, y, grid_mapping)
295
+
296
+ if grid is not None:
297
+ self._cache[(x.name, y.name, dim_vars)] = grid
298
+ return grid
296
299
 
297
300
  LOG.error("Could not fine a candidate for 'grid_mapping'")
298
301
  raise NotImplementedError(f"Unstructured grid {x.name} {y.name}")
@@ -340,12 +343,14 @@ class DefaultCoordinateGuesser(CoordinateGuesser):
340
343
  if standard_name == "time":
341
344
  return TimeCoordinate(c)
342
345
 
343
- if name == "time":
346
+ # That is the output of `cfgrib` for forecasts
347
+ if name == "time" and standard_name != "forecast_reference_time":
344
348
  return TimeCoordinate(c)
345
349
 
346
350
  def _is_date(self, c, *, axis, name, long_name, standard_name, units):
347
351
  if standard_name == "forecast_reference_time":
348
352
  return DateCoordinate(c)
353
+
349
354
  if name == "forecast_reference_time":
350
355
  return DateCoordinate(c)
351
356
 
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  import logging
@@ -100,6 +100,7 @@ class ProjectionGrid(XYGrid):
100
100
  data_crs = CRS.from_cf(self.projection)
101
101
  else:
102
102
  data_crs = self.projection
103
+
103
104
  wgs84_crs = CRS.from_epsg(4326) # WGS84
104
105
 
105
106
  return Transformer.from_crs(data_crs, wgs84_crs, always_xy=True)
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  import logging
11
12
  from functools import cached_property
@@ -1,17 +1,20 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  import datetime
12
+ import logging
12
13
 
13
14
  from anemoi.utils.dates import as_datetime
14
15
 
16
+ LOG = logging.getLogger(__name__)
17
+
15
18
 
16
19
  class Time:
17
20
 
@@ -36,7 +39,28 @@ class Time:
36
39
  if len(date_coordinate) == 1 and len(time_coordinate) == 0 and len(step_coordinate) == 1:
37
40
  return ForecastFromBaseTimeAndDate(date_coordinate[0], step_coordinate[0])
38
41
 
39
- raise NotImplementedError(f"{date_coordinate=} {time_coordinate=} {step_coordinate=}")
42
+ if len(date_coordinate) == 1 and len(time_coordinate) == 1 and len(step_coordinate) == 1:
43
+ return ForecastFromValidTimeAndStep(time_coordinate[0], step_coordinate[0], date_coordinate[0])
44
+
45
+ LOG.error("")
46
+ LOG.error(f"{len(date_coordinate)} date_coordinate")
47
+ for c in date_coordinate:
48
+ LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
49
+ # LOG.error(' %s', c.variable)
50
+
51
+ LOG.error("")
52
+ LOG.error(f"{len(time_coordinate)} time_coordinate")
53
+ for c in time_coordinate:
54
+ LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
55
+ # LOG.error(' %s', c.variable)
56
+
57
+ LOG.error("")
58
+ LOG.error(f"{len(step_coordinate)} step_coordinate")
59
+ for c in step_coordinate:
60
+ LOG.error(" %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
61
+ # LOG.error(' %s', c.variable)
62
+
63
+ raise NotImplementedError(f"{len(date_coordinate)=} {len(time_coordinate)=} {len(step_coordinate)=}")
40
64
 
41
65
 
42
66
  class Constant(Time):
@@ -62,9 +86,10 @@ class Analysis(Time):
62
86
 
63
87
  class ForecastFromValidTimeAndStep(Time):
64
88
 
65
- def __init__(self, time_coordinate, step_coordinate):
89
+ def __init__(self, time_coordinate, step_coordinate, date_coordinate=None):
66
90
  self.time_coordinate_name = time_coordinate.variable.name
67
91
  self.step_coordinate_name = step_coordinate.variable.name
92
+ self.date_coordinate_name = date_coordinate.variable.name if date_coordinate else None
68
93
 
69
94
  def fill_time_metadata(self, coords_values, metadata):
70
95
  valid_datetime = coords_values[self.time_coordinate_name]
@@ -79,6 +104,16 @@ class ForecastFromValidTimeAndStep(Time):
79
104
  metadata["date"] = as_datetime(base_datetime).strftime("%Y%m%d")
80
105
  metadata["time"] = as_datetime(base_datetime).strftime("%H%M")
81
106
  metadata["step"] = int(hours)
107
+
108
+ # When date is present, it should be compatible with time and step
109
+
110
+ if self.date_coordinate_name is not None:
111
+ # Not sure that this is the correct assumption
112
+ assert coords_values[self.date_coordinate_name] == base_datetime, (
113
+ coords_values[self.date_coordinate_name],
114
+ base_datetime,
115
+ )
116
+
82
117
  return valid_datetime
83
118
 
84
119
 
@@ -1,11 +1,12 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
+
9
10
 
10
11
  import logging
11
12
  import math
@@ -82,9 +83,8 @@ class Variable:
82
83
  )
83
84
 
84
85
  def __getitem__(self, i):
85
- """
86
- Get a 2D field from the variable
87
- """
86
+ """Get a 2D field from the variable"""
87
+
88
88
  if i >= self.length:
89
89
  raise IndexError(i)
90
90
 
@@ -117,7 +117,7 @@ class Variable:
117
117
 
118
118
  variable = Variable(
119
119
  ds=self.ds,
120
- var=self.variable.isel({k: i}),
120
+ variable=self.variable.isel({k: i}),
121
121
  coordinates=coordinates,
122
122
  grid=self.grid,
123
123
  time=self.time,
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  from earthkit.data.core.fieldlist import MultiFieldList
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  from .xarray import load_many
@@ -1,11 +1,11 @@
1
- # (C) Copyright 2024 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
9
 
10
10
 
11
11
  from earthkit.data.core.fieldlist import MultiFieldList
@@ -1,28 +1,14 @@
1
- # (C) Copyright 2023 ECMWF.
1
+ # (C) Copyright 2024 Anemoi contributors.
2
2
  #
3
3
  # This software is licensed under the terms of the Apache Licence Version 2.0
4
4
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5
+ #
5
6
  # In applying this licence, ECMWF does not waive the privileges and immunities
6
7
  # granted to it by virtue of its status as an intergovernmental organisation
7
8
  # nor does it submit to any jurisdiction.
8
- #
9
- import datetime
10
- import itertools
9
+
11
10
  import logging
12
- import math
13
- import time
14
- from collections import defaultdict
15
11
  from copy import deepcopy
16
- from functools import cached_property
17
- from functools import wraps
18
-
19
- import numpy as np
20
- from anemoi.utils.dates import as_datetime as as_datetime
21
- from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
22
-
23
- from anemoi.datasets.dates import DatesProvider as DatesProvider
24
- from anemoi.datasets.fields import FieldArray as FieldArray
25
- from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
26
12
 
27
13
  from .trace import trace_select
28
14