anemoi-datasets 0.4.4__py3-none-any.whl → 0.4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anemoi/datasets/_version.py +2 -2
- anemoi/datasets/commands/cleanup.py +44 -0
- anemoi/datasets/commands/create.py +50 -20
- anemoi/datasets/commands/finalise-additions.py +45 -0
- anemoi/datasets/commands/finalise.py +39 -0
- anemoi/datasets/commands/init-additions.py +45 -0
- anemoi/datasets/commands/init.py +67 -0
- anemoi/datasets/commands/inspect.py +1 -1
- anemoi/datasets/commands/load-additions.py +47 -0
- anemoi/datasets/commands/load.py +47 -0
- anemoi/datasets/commands/patch.py +39 -0
- anemoi/datasets/create/__init__.py +961 -146
- anemoi/datasets/create/check.py +5 -3
- anemoi/datasets/create/config.py +53 -2
- anemoi/datasets/create/functions/sources/xarray/__init__.py +12 -2
- anemoi/datasets/create/functions/sources/xarray/coordinates.py +7 -0
- anemoi/datasets/create/functions/sources/xarray/field.py +1 -1
- anemoi/datasets/create/functions/sources/xarray/fieldlist.py +0 -2
- anemoi/datasets/create/functions/sources/xarray/flavour.py +21 -1
- anemoi/datasets/create/functions/sources/xarray/metadata.py +27 -29
- anemoi/datasets/create/functions/sources/xarray/time.py +63 -30
- anemoi/datasets/create/functions/sources/xarray/variable.py +15 -38
- anemoi/datasets/create/input.py +23 -22
- anemoi/datasets/create/statistics/__init__.py +39 -23
- anemoi/datasets/create/utils.py +3 -2
- anemoi/datasets/data/__init__.py +1 -0
- anemoi/datasets/data/concat.py +46 -2
- anemoi/datasets/data/dataset.py +109 -34
- anemoi/datasets/data/forwards.py +17 -8
- anemoi/datasets/data/grids.py +17 -3
- anemoi/datasets/data/interpolate.py +133 -0
- anemoi/datasets/data/misc.py +56 -66
- anemoi/datasets/data/missing.py +240 -0
- anemoi/datasets/data/select.py +7 -1
- anemoi/datasets/data/stores.py +3 -3
- anemoi/datasets/data/subset.py +47 -5
- anemoi/datasets/data/unchecked.py +20 -22
- anemoi/datasets/data/xy.py +125 -0
- anemoi/datasets/dates/__init__.py +13 -66
- anemoi/datasets/dates/groups.py +2 -2
- anemoi/datasets/grids.py +66 -48
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/METADATA +5 -5
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/RECORD +47 -37
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/WHEEL +1 -1
- anemoi/datasets/create/loaders.py +0 -936
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/LICENSE +0 -0
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/entry_points.txt +0 -0
- {anemoi_datasets-0.4.4.dist-info → anemoi_datasets-0.4.5.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
|
|
2
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
3
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
4
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
5
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
6
|
+
# nor does it submit to any jurisdiction.
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from functools import cached_property
|
|
10
|
+
|
|
11
|
+
from .debug import Node
|
|
12
|
+
from .forwards import Combined
|
|
13
|
+
from .misc import _auto_adjust
|
|
14
|
+
from .misc import _open
|
|
15
|
+
|
|
16
|
+
LOG = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ZipBase(Combined):
|
|
20
|
+
|
|
21
|
+
def swap_with_parent(self, parent):
|
|
22
|
+
new_parents = [parent.clone(ds) for ds in self.datasets]
|
|
23
|
+
return self.clone(new_parents)
|
|
24
|
+
|
|
25
|
+
def clone(self, datasets):
|
|
26
|
+
return self.__class__(datasets)
|
|
27
|
+
|
|
28
|
+
def tree(self):
|
|
29
|
+
return Node(self, [d.tree() for d in self.datasets])
|
|
30
|
+
|
|
31
|
+
def __len__(self):
|
|
32
|
+
return min(len(d) for d in self.datasets)
|
|
33
|
+
|
|
34
|
+
def __getitem__(self, n):
|
|
35
|
+
return tuple(d[n] for d in self.datasets)
|
|
36
|
+
|
|
37
|
+
def check_same_resolution(self, d1, d2):
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
def check_same_grid(self, d1, d2):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
def check_same_variables(self, d1, d2):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
@cached_property
|
|
47
|
+
def missing(self):
|
|
48
|
+
result = set()
|
|
49
|
+
for d in self.datasets:
|
|
50
|
+
result = result | d.missing
|
|
51
|
+
return result
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def shape(self):
|
|
55
|
+
return tuple(d.shape for d in self.datasets)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def field_shape(self):
|
|
59
|
+
return tuple(d.shape for d in self.datasets)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def latitudes(self):
|
|
63
|
+
return tuple(d.latitudes for d in self.datasets)
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def longitudes(self):
|
|
67
|
+
return tuple(d.longitudes for d in self.datasets)
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def dtype(self):
|
|
71
|
+
return tuple(d.dtype for d in self.datasets)
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def grids(self):
|
|
75
|
+
return tuple(d.grids for d in self.datasets)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def statistics(self):
|
|
79
|
+
return tuple(d.statistics for d in self.datasets)
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def resolution(self):
|
|
83
|
+
return tuple(d.resolution for d in self.datasets)
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def name_to_index(self):
|
|
87
|
+
return tuple(d.name_to_index for d in self.datasets)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class Zip(ZipBase):
|
|
91
|
+
pass
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class XY(ZipBase):
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def xy_factory(args, kwargs):
|
|
99
|
+
|
|
100
|
+
if "xy" in kwargs:
|
|
101
|
+
xy = kwargs.pop("xy")
|
|
102
|
+
else:
|
|
103
|
+
xy = [kwargs.pop("x"), kwargs.pop("y")]
|
|
104
|
+
|
|
105
|
+
assert len(args) == 0
|
|
106
|
+
assert isinstance(xy, (list, tuple))
|
|
107
|
+
|
|
108
|
+
datasets = [_open(e) for e in xy]
|
|
109
|
+
datasets, kwargs = _auto_adjust(datasets, kwargs)
|
|
110
|
+
|
|
111
|
+
assert len(datasets) == 2
|
|
112
|
+
|
|
113
|
+
return XY(datasets)._subset(**kwargs)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def zip_factory(args, kwargs):
|
|
117
|
+
|
|
118
|
+
zip = kwargs.pop("zip")
|
|
119
|
+
assert len(args) == 0
|
|
120
|
+
assert isinstance(zip, (list, tuple))
|
|
121
|
+
|
|
122
|
+
datasets = [_open(e) for e in zip]
|
|
123
|
+
datasets, kwargs = _auto_adjust(datasets, kwargs)
|
|
124
|
+
|
|
125
|
+
return Zip(datasets)._subset(**kwargs)
|
|
@@ -9,64 +9,10 @@
|
|
|
9
9
|
import datetime
|
|
10
10
|
import warnings
|
|
11
11
|
|
|
12
|
+
# from anemoi.utils.dates import as_datetime
|
|
12
13
|
from anemoi.utils.dates import as_datetime
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def _compress_dates(dates):
|
|
16
|
-
dates = sorted(dates)
|
|
17
|
-
if len(dates) < 3:
|
|
18
|
-
yield dates
|
|
19
|
-
return
|
|
20
|
-
|
|
21
|
-
prev = first = dates.pop(0)
|
|
22
|
-
curr = dates.pop(0)
|
|
23
|
-
delta = curr - prev
|
|
24
|
-
while curr - prev == delta:
|
|
25
|
-
prev = curr
|
|
26
|
-
if not dates:
|
|
27
|
-
break
|
|
28
|
-
curr = dates.pop(0)
|
|
29
|
-
|
|
30
|
-
yield (first, prev, delta)
|
|
31
|
-
if dates:
|
|
32
|
-
yield from _compress_dates([curr] + dates)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def compress_dates(dates):
|
|
36
|
-
dates = [as_datetime(_) for _ in dates]
|
|
37
|
-
result = []
|
|
38
|
-
|
|
39
|
-
for n in _compress_dates(dates):
|
|
40
|
-
if isinstance(n, list):
|
|
41
|
-
result.extend([str(_) for _ in n])
|
|
42
|
-
else:
|
|
43
|
-
result.append(" ".join([str(n[0]), "to", str(n[1]), "by", str(n[2])]))
|
|
44
|
-
|
|
45
|
-
return result
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def print_dates(dates):
|
|
49
|
-
print(compress_dates(dates))
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def no_time_zone(date):
|
|
53
|
-
return date.replace(tzinfo=None)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def frequency_to_hours(frequency):
|
|
57
|
-
if isinstance(frequency, int):
|
|
58
|
-
return frequency
|
|
59
|
-
assert isinstance(frequency, str), (type(frequency), frequency)
|
|
60
|
-
|
|
61
|
-
unit = frequency[-1].lower()
|
|
62
|
-
v = int(frequency[:-1])
|
|
63
|
-
return {"h": v, "d": v * 24}[unit]
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def normalize_date(x):
|
|
67
|
-
if isinstance(x, str):
|
|
68
|
-
return no_time_zone(datetime.datetime.fromisoformat(x))
|
|
69
|
-
return x
|
|
14
|
+
from anemoi.utils.dates import frequency_to_timedelta
|
|
15
|
+
from anemoi.utils.humanize import print_dates
|
|
70
16
|
|
|
71
17
|
|
|
72
18
|
def extend(x):
|
|
@@ -79,15 +25,15 @@ def extend(x):
|
|
|
79
25
|
if isinstance(x, str):
|
|
80
26
|
if "/" in x:
|
|
81
27
|
start, end, step = x.split("/")
|
|
82
|
-
start =
|
|
83
|
-
end =
|
|
84
|
-
step =
|
|
28
|
+
start = as_datetime(start)
|
|
29
|
+
end = as_datetime(end)
|
|
30
|
+
step = frequency_to_timedelta(step)
|
|
85
31
|
while start <= end:
|
|
86
32
|
yield start
|
|
87
33
|
start += datetime.timedelta(hours=step)
|
|
88
34
|
return
|
|
89
35
|
|
|
90
|
-
yield
|
|
36
|
+
yield as_datetime(x)
|
|
91
37
|
|
|
92
38
|
|
|
93
39
|
class Dates:
|
|
@@ -145,7 +91,7 @@ class Dates:
|
|
|
145
91
|
|
|
146
92
|
class ValuesDates(Dates):
|
|
147
93
|
def __init__(self, values, **kwargs):
|
|
148
|
-
self.values = sorted([
|
|
94
|
+
self.values = sorted([as_datetime(_) for _ in values])
|
|
149
95
|
super().__init__(**kwargs)
|
|
150
96
|
|
|
151
97
|
def __repr__(self):
|
|
@@ -157,7 +103,8 @@ class ValuesDates(Dates):
|
|
|
157
103
|
|
|
158
104
|
class StartEndDates(Dates):
|
|
159
105
|
def __init__(self, start, end, frequency=1, months=None, **kwargs):
|
|
160
|
-
frequency =
|
|
106
|
+
frequency = frequency_to_timedelta(frequency)
|
|
107
|
+
assert isinstance(frequency, datetime.timedelta), frequency
|
|
161
108
|
|
|
162
109
|
def _(x):
|
|
163
110
|
if isinstance(x, str):
|
|
@@ -173,13 +120,13 @@ class StartEndDates(Dates):
|
|
|
173
120
|
if isinstance(end, datetime.date) and not isinstance(end, datetime.datetime):
|
|
174
121
|
end = datetime.datetime(end.year, end.month, end.day)
|
|
175
122
|
|
|
176
|
-
start =
|
|
177
|
-
end =
|
|
123
|
+
start = as_datetime(start)
|
|
124
|
+
end = as_datetime(end)
|
|
178
125
|
|
|
179
126
|
# if end <= start:
|
|
180
127
|
# raise ValueError(f"End date {end} must be after start date {start}")
|
|
181
128
|
|
|
182
|
-
increment =
|
|
129
|
+
increment = frequency
|
|
183
130
|
|
|
184
131
|
self.start = start
|
|
185
132
|
self.end = end
|
anemoi/datasets/dates/groups.py
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import itertools
|
|
10
10
|
|
|
11
11
|
from anemoi.datasets.dates import Dates
|
|
12
|
-
from anemoi.datasets.dates import
|
|
12
|
+
from anemoi.datasets.dates import as_datetime
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class Groups:
|
|
@@ -67,7 +67,7 @@ class Groups:
|
|
|
67
67
|
|
|
68
68
|
class Filter:
|
|
69
69
|
def __init__(self, missing):
|
|
70
|
-
self.missing = [
|
|
70
|
+
self.missing = [as_datetime(m) for m in missing]
|
|
71
71
|
|
|
72
72
|
def __call__(self, dates):
|
|
73
73
|
return [d for d in dates if d not in self.missing]
|
anemoi/datasets/grids.py
CHANGED
|
@@ -7,41 +7,65 @@
|
|
|
7
7
|
# nor does it submit to any jurisdiction.
|
|
8
8
|
#
|
|
9
9
|
|
|
10
|
+
import logging
|
|
11
|
+
|
|
10
12
|
import numpy as np
|
|
11
13
|
|
|
14
|
+
LOG = logging.getLogger(__name__)
|
|
15
|
+
|
|
12
16
|
|
|
13
17
|
def plot_mask(path, mask, lats, lons, global_lats, global_lons):
|
|
14
18
|
import matplotlib.pyplot as plt
|
|
15
19
|
|
|
16
|
-
middle = (np.amin(lons) + np.amax(lons)) / 2
|
|
17
|
-
print("middle", middle)
|
|
18
20
|
s = 1
|
|
19
21
|
|
|
20
|
-
# gmiddle = (np.amin(global_lons)+ np.amax(global_lons))/2
|
|
21
|
-
|
|
22
|
-
# print('gmiddle', gmiddle)
|
|
23
|
-
# global_lons = global_lons-gmiddle+middle
|
|
24
22
|
global_lons[global_lons >= 180] -= 360
|
|
25
23
|
|
|
26
24
|
plt.figure(figsize=(10, 5))
|
|
27
25
|
plt.scatter(global_lons, global_lats, s=s, marker="o", c="r")
|
|
28
|
-
|
|
26
|
+
if isinstance(path, str):
|
|
27
|
+
plt.savefig(path + "-global.png")
|
|
29
28
|
|
|
30
29
|
plt.figure(figsize=(10, 5))
|
|
31
30
|
plt.scatter(global_lons[mask], global_lats[mask], s=s, c="k")
|
|
32
|
-
|
|
31
|
+
if isinstance(path, str):
|
|
32
|
+
plt.savefig(path + "-cutout.png")
|
|
33
33
|
|
|
34
34
|
plt.figure(figsize=(10, 5))
|
|
35
35
|
plt.scatter(lons, lats, s=s)
|
|
36
|
-
|
|
36
|
+
if isinstance(path, str):
|
|
37
|
+
plt.savefig(path + "-lam.png")
|
|
37
38
|
# plt.scatter(lons, lats, s=0.01)
|
|
38
39
|
|
|
39
40
|
plt.figure(figsize=(10, 5))
|
|
40
41
|
plt.scatter(global_lons[mask], global_lats[mask], s=s, c="r")
|
|
41
42
|
plt.scatter(lons, lats, s=s)
|
|
42
|
-
|
|
43
|
+
if isinstance(path, str):
|
|
44
|
+
plt.savefig(path + "-both.png")
|
|
43
45
|
# plt.scatter(lons, lats, s=0.01)
|
|
44
46
|
|
|
47
|
+
plt.figure(figsize=(10, 5))
|
|
48
|
+
plt.scatter(global_lons[mask], global_lats[mask], s=s, c="r")
|
|
49
|
+
plt.scatter(lons, lats, s=s)
|
|
50
|
+
plt.xlim(np.amin(lons) - 1, np.amax(lons) + 1)
|
|
51
|
+
plt.ylim(np.amin(lats) - 1, np.amax(lats) + 1)
|
|
52
|
+
if isinstance(path, str):
|
|
53
|
+
plt.savefig(path + "-both-zoomed.png")
|
|
54
|
+
|
|
55
|
+
plt.figure(figsize=(10, 5))
|
|
56
|
+
plt.scatter(global_lons[mask], global_lats[mask], s=s, c="r")
|
|
57
|
+
plt.xlim(np.amin(lons) - 1, np.amax(lons) + 1)
|
|
58
|
+
plt.ylim(np.amin(lats) - 1, np.amax(lats) + 1)
|
|
59
|
+
if isinstance(path, str):
|
|
60
|
+
plt.savefig(path + "-global-zoomed.png")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def xyz_to_latlon(x, y, z):
|
|
64
|
+
return (
|
|
65
|
+
np.rad2deg(np.arcsin(np.minimum(1.0, np.maximum(-1.0, z)))),
|
|
66
|
+
np.rad2deg(np.arctan2(y, x)),
|
|
67
|
+
)
|
|
68
|
+
|
|
45
69
|
|
|
46
70
|
def latlon_to_xyz(lat, lon, radius=1.0):
|
|
47
71
|
# https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
|
|
@@ -121,6 +145,7 @@ def cutout_mask(
|
|
|
121
145
|
global_lats,
|
|
122
146
|
global_lons,
|
|
123
147
|
cropping_distance=2.0,
|
|
148
|
+
neighbours=5,
|
|
124
149
|
min_distance_km=None,
|
|
125
150
|
plot=None,
|
|
126
151
|
):
|
|
@@ -164,58 +189,52 @@ def cutout_mask(
|
|
|
164
189
|
xyx = latlon_to_xyz(lats, lons)
|
|
165
190
|
lam_points = np.array(xyx).transpose()
|
|
166
191
|
|
|
167
|
-
|
|
168
|
-
kdtree = KDTree(lam_points)
|
|
169
|
-
distances, indices = kdtree.query(global_points, k=3)
|
|
170
|
-
|
|
171
|
-
if min_distance_km is not None:
|
|
192
|
+
if isinstance(min_distance_km, (int, float)):
|
|
172
193
|
min_distance = min_distance_km / 6371.0
|
|
173
194
|
else:
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
centre = np.mean(lats), np.mean(lons)
|
|
183
|
-
centre_xyz = np.array(latlon_to_xyz(*centre))
|
|
184
|
-
|
|
185
|
-
pt1 = np.array(latlon_to_xyz(centre[0] + min_dlats, centre[1]))
|
|
186
|
-
pt2 = np.array(latlon_to_xyz(centre[0], centre[1] + min_dlons))
|
|
187
|
-
min_distance = (
|
|
188
|
-
min(
|
|
189
|
-
np.linalg.norm(pt1 - centre_xyz),
|
|
190
|
-
np.linalg.norm(pt2 - centre_xyz),
|
|
191
|
-
)
|
|
192
|
-
/ 2.0
|
|
193
|
-
)
|
|
195
|
+
points = {"lam": lam_points, "global": global_points, None: global_points}[min_distance_km]
|
|
196
|
+
distances, _ = KDTree(points).query(points, k=2)
|
|
197
|
+
min_distance = np.min(distances[:, 1])
|
|
198
|
+
|
|
199
|
+
LOG.info(f"cutout_mask using min_distance = {min_distance * 6371.0} km")
|
|
200
|
+
|
|
201
|
+
# Use a KDTree to find the nearest points
|
|
202
|
+
distances, indices = KDTree(lam_points).query(global_points, k=neighbours)
|
|
194
203
|
|
|
204
|
+
# Centre of the Earth
|
|
195
205
|
zero = np.array([0.0, 0.0, 0.0])
|
|
196
|
-
|
|
206
|
+
|
|
207
|
+
# After the loop, 'inside_lam' will contain a list point to EXCLUDE
|
|
208
|
+
inside_lam = []
|
|
209
|
+
|
|
197
210
|
for i, (global_point, distance, index) in enumerate(zip(global_points, distances, indices)):
|
|
198
|
-
t = Triangle3D(lam_points[index[0]], lam_points[index[1]], lam_points[index[2]])
|
|
199
|
-
# distance = np.min(distance)
|
|
200
|
-
# The point is inside the triangle if the intersection with the ray
|
|
201
|
-
# from the point to the centre of the Earth is not None
|
|
202
|
-
# (the direction of the ray is not important)
|
|
203
211
|
|
|
204
|
-
|
|
212
|
+
# We check more than one triangle in case te global point
|
|
213
|
+
# is near the edge of triangle, (the lam point and global points are colinear)
|
|
214
|
+
|
|
215
|
+
inside = False
|
|
216
|
+
for j in range(neighbours):
|
|
217
|
+
t = Triangle3D(
|
|
218
|
+
lam_points[index[j]], lam_points[index[(j + 1) % neighbours]], lam_points[index[(j + 2) % neighbours]]
|
|
219
|
+
)
|
|
220
|
+
inside = t.intersect(zero, global_point)
|
|
221
|
+
if inside:
|
|
222
|
+
break
|
|
223
|
+
|
|
205
224
|
close = np.min(distance) <= min_distance
|
|
206
225
|
|
|
207
|
-
|
|
226
|
+
inside_lam.append(inside or close)
|
|
208
227
|
|
|
209
228
|
j = 0
|
|
210
|
-
|
|
229
|
+
inside_lam = np.array(inside_lam)
|
|
211
230
|
for i, m in enumerate(mask):
|
|
212
231
|
if not m:
|
|
213
232
|
continue
|
|
214
233
|
|
|
215
|
-
mask[i] =
|
|
234
|
+
mask[i] = inside_lam[j]
|
|
216
235
|
j += 1
|
|
217
236
|
|
|
218
|
-
assert j == len(
|
|
237
|
+
assert j == len(inside_lam)
|
|
219
238
|
|
|
220
239
|
# Invert the mask, so we have only the points outside the cutout
|
|
221
240
|
mask = ~mask
|
|
@@ -271,8 +290,7 @@ def thinning_mask(
|
|
|
271
290
|
points = np.array(xyx).transpose()
|
|
272
291
|
|
|
273
292
|
# Use a KDTree to find the nearest points
|
|
274
|
-
|
|
275
|
-
_, indices = kdtree.query(global_points, k=1)
|
|
293
|
+
_, indices = KDTree(points).query(global_points, k=1)
|
|
276
294
|
|
|
277
295
|
return np.array([i for i in indices])
|
|
278
296
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: anemoi-datasets
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.5
|
|
4
4
|
Summary: A package to hold various functions to support training of ML models on ECMWF data.
|
|
5
5
|
Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
|
|
6
6
|
License: Apache License
|
|
@@ -223,7 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
|
223
223
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
224
224
|
Requires-Python: >=3.9
|
|
225
225
|
License-File: LICENSE
|
|
226
|
-
Requires-Dist: anemoi-utils[provenance] >=0.3.
|
|
226
|
+
Requires-Dist: anemoi-utils[provenance] >=0.3.15
|
|
227
227
|
Requires-Dist: numpy
|
|
228
228
|
Requires-Dist: pyyaml
|
|
229
229
|
Requires-Dist: semantic-version
|
|
@@ -235,7 +235,7 @@ Requires-Dist: boto3 ; extra == 'all'
|
|
|
235
235
|
Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'all'
|
|
236
236
|
Requires-Dist: earthkit-geo >=0.2 ; extra == 'all'
|
|
237
237
|
Requires-Dist: earthkit-meteo ; extra == 'all'
|
|
238
|
-
Requires-Dist:
|
|
238
|
+
Requires-Dist: eccodes >=2.37 ; extra == 'all'
|
|
239
239
|
Requires-Dist: entrypoints ; extra == 'all'
|
|
240
240
|
Requires-Dist: gcsfs ; extra == 'all'
|
|
241
241
|
Requires-Dist: kerchunk ; extra == 'all'
|
|
@@ -246,7 +246,7 @@ Provides-Extra: create
|
|
|
246
246
|
Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'create'
|
|
247
247
|
Requires-Dist: earthkit-geo >=0.2 ; extra == 'create'
|
|
248
248
|
Requires-Dist: earthkit-meteo ; extra == 'create'
|
|
249
|
-
Requires-Dist:
|
|
249
|
+
Requires-Dist: eccodes >=2.37 ; extra == 'create'
|
|
250
250
|
Requires-Dist: entrypoints ; extra == 'create'
|
|
251
251
|
Requires-Dist: pyproj ; extra == 'create'
|
|
252
252
|
Provides-Extra: dev
|
|
@@ -255,7 +255,7 @@ Requires-Dist: boto3 ; extra == 'dev'
|
|
|
255
255
|
Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'dev'
|
|
256
256
|
Requires-Dist: earthkit-geo >=0.2 ; extra == 'dev'
|
|
257
257
|
Requires-Dist: earthkit-meteo ; extra == 'dev'
|
|
258
|
-
Requires-Dist:
|
|
258
|
+
Requires-Dist: eccodes >=2.37 ; extra == 'dev'
|
|
259
259
|
Requires-Dist: entrypoints ; extra == 'dev'
|
|
260
260
|
Requires-Dist: gcsfs ; extra == 'dev'
|
|
261
261
|
Requires-Dist: kerchunk ; extra == 'dev'
|
|
@@ -1,27 +1,34 @@
|
|
|
1
1
|
anemoi/datasets/__init__.py,sha256=Z1gqZWhecLcT0RZQqYBLlz01MUlUZd0kWEj_RavbITM,782
|
|
2
2
|
anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
|
|
3
|
-
anemoi/datasets/_version.py,sha256=
|
|
4
|
-
anemoi/datasets/grids.py,sha256=
|
|
3
|
+
anemoi/datasets/_version.py,sha256=XyejXnBNtqNdR37CiSnJ022K8vUHKAlzN9lbcpmmftA,411
|
|
4
|
+
anemoi/datasets/grids.py,sha256=xgvIbpMGuN2GKi2wIBhOLEMzj940nY9PH-toD0rCmPo,8980
|
|
5
5
|
anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
|
|
6
|
+
anemoi/datasets/commands/cleanup.py,sha256=_BkzGPUgvSqnuleymBfBx-sIyIM55hjK61m-v7yK0T8,1062
|
|
6
7
|
anemoi/datasets/commands/compare.py,sha256=svEhyR7pOS1847_RJr1I6vF7ZDPB9AVlcrhy_gxQVms,3263
|
|
7
8
|
anemoi/datasets/commands/copy.py,sha256=SxAeN51owyN5gwtwpt30xhJSIJRlJb9YOUt_4K4m-D8,11780
|
|
8
|
-
anemoi/datasets/commands/create.py,sha256=
|
|
9
|
-
anemoi/datasets/commands/
|
|
9
|
+
anemoi/datasets/commands/create.py,sha256=jdwgzUvZBkXROkJ-LSbi28mVWnnZ5F1xr0IOciktfrU,5012
|
|
10
|
+
anemoi/datasets/commands/finalise-additions.py,sha256=876K37hVjslHJiu9VXZfre4YJhS2_t9rLhmNWqlKGGk,1158
|
|
11
|
+
anemoi/datasets/commands/finalise.py,sha256=oPul5za7E__eJqkf5fRwvdL_0n2nG5Xk3JraRZQe64k,913
|
|
12
|
+
anemoi/datasets/commands/init-additions.py,sha256=K3eKH5V_6ERiBKKyqIUuI1cfvAsjWV9ZAFzbtjIDyjs,1146
|
|
13
|
+
anemoi/datasets/commands/init.py,sha256=7BpWaZ4gzMMiRqD9djHRciQNa0W2R4cOzoy5R-UJ0ck,2033
|
|
14
|
+
anemoi/datasets/commands/inspect.py,sha256=g2H_Ey47szeRp6sfaXkbAuk2JWW-LL4GXpssJIsWKVo,18675
|
|
15
|
+
anemoi/datasets/commands/load-additions.py,sha256=Tg4FX0xealmwyB--zv03ZBrqTAGddkT0rKrHlcu4VW0,1234
|
|
16
|
+
anemoi/datasets/commands/load.py,sha256=3jhL3DKUg07C8D6L8RsRWciq8korYsJ40_9atp-p68U,1304
|
|
17
|
+
anemoi/datasets/commands/patch.py,sha256=gHY16r46GHxARItaAUHVjKSOm0Q7ZyZvSZmXqg1Wnig,841
|
|
10
18
|
anemoi/datasets/commands/scan.py,sha256=MaTdne4JrtlqO3LhOUr43DZhZ6O-RZwC7uQ7C6PG7Os,2910
|
|
11
19
|
anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
20
|
anemoi/datasets/compute/recentre.py,sha256=CMvRxTMv2H1hF1KqaZHT3Maa9P01GtsgAxr1YHbvH_o,4939
|
|
13
|
-
anemoi/datasets/create/__init__.py,sha256=
|
|
14
|
-
anemoi/datasets/create/check.py,sha256=
|
|
21
|
+
anemoi/datasets/create/__init__.py,sha256=8eYVI0QAkdAIdxVDrqQAY34yDlLAAsr92DL-e_nhHCM,35163
|
|
22
|
+
anemoi/datasets/create/check.py,sha256=9If6iAXILExjzX1YAWCDekV821BquHdjDaUSb_S_gp0,6008
|
|
15
23
|
anemoi/datasets/create/chunks.py,sha256=1Inh3sIBn-2sNguErb-BsLns6W_HtDiOJAjIb29lp-U,2442
|
|
16
|
-
anemoi/datasets/create/config.py,sha256=
|
|
17
|
-
anemoi/datasets/create/input.py,sha256=
|
|
18
|
-
anemoi/datasets/create/loaders.py,sha256=dEGj7FxKHgxXTVBSHB-7kzOK7sjbiWOuzE2y5eDXla0,33130
|
|
24
|
+
anemoi/datasets/create/config.py,sha256=ojynLQTX7BmJRZ18rhjUTmuMOh0rYAx9M5d6gY8wRH0,8872
|
|
25
|
+
anemoi/datasets/create/input.py,sha256=LArEOIxa6Y42rneF0VUva25zOvks_BwVDfBNOSERpYA,33924
|
|
19
26
|
anemoi/datasets/create/patch.py,sha256=c8cWoqxFzcY9mKREosLjuQCUeJMJL6sbNNkoVvhCZDA,3800
|
|
20
27
|
anemoi/datasets/create/persistent.py,sha256=wfNnNSY8VLO2EyEOQL9umI3sbQqETjJP7bPCiVF6Cko,4291
|
|
21
28
|
anemoi/datasets/create/size.py,sha256=k4COEjs3wud0oKHH5P3n8Fap35xddXs002ucPjpBC88,1040
|
|
22
29
|
anemoi/datasets/create/template.py,sha256=1t8EKGQcZGFUpgQw9a9oEy6ZWlfnow5e0vs1SOelUNc,3148
|
|
23
30
|
anemoi/datasets/create/trace.py,sha256=J-8jDy28wNZa4aSV1KIQMwc1KolcoH3R2xjLl-_eLzM,2183
|
|
24
|
-
anemoi/datasets/create/utils.py,sha256=
|
|
31
|
+
anemoi/datasets/create/utils.py,sha256=mgin7LF-JkfXMr22IIIEQjfUHFkwPwMJHca4Fek3F6Q,2639
|
|
25
32
|
anemoi/datasets/create/writer.py,sha256=G1qAPvdn8anGnpWYhvSSP4u3Km_tHKPdMXm0G4skKSk,1379
|
|
26
33
|
anemoi/datasets/create/zarr.py,sha256=Pb57mZn5s4JTA1o6_satrvG7C8XQhgPFhpTGvlaC_kg,5340
|
|
27
34
|
anemoi/datasets/create/functions/__init__.py,sha256=5HmelLkXDjFOhNhX0Z78aV3ZlW2txiJliJwT4jfLEN4,945
|
|
@@ -47,40 +54,43 @@ anemoi/datasets/create/functions/sources/tendencies.py,sha256=4hFPGU51KnxXp-KteS
|
|
|
47
54
|
anemoi/datasets/create/functions/sources/xarray_kerchunk.py,sha256=JsuGjTs5BQdJIPRI8TuUNde680UQkT4GbUhOt6wYy38,1432
|
|
48
55
|
anemoi/datasets/create/functions/sources/xarray_zarr.py,sha256=qI_fEKo28hu_B_qMPx67gigysw8qw5ePMU9P0wbDcIc,531
|
|
49
56
|
anemoi/datasets/create/functions/sources/zenodo.py,sha256=n7_sfZHJVDjVnSwc07muO9kS7TwOM4S8DIlJf5pnDiU,1237
|
|
50
|
-
anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=
|
|
51
|
-
anemoi/datasets/create/functions/sources/xarray/coordinates.py,sha256=
|
|
52
|
-
anemoi/datasets/create/functions/sources/xarray/field.py,sha256=
|
|
53
|
-
anemoi/datasets/create/functions/sources/xarray/fieldlist.py,sha256=
|
|
54
|
-
anemoi/datasets/create/functions/sources/xarray/flavour.py,sha256=
|
|
57
|
+
anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=XcP072AgDX_aiK3t_rWM-C3MnAuX-mhYO5-wSLyaxPo,2805
|
|
58
|
+
anemoi/datasets/create/functions/sources/xarray/coordinates.py,sha256=iwnGIinuLoP0jk2s4ro9_ihJ6CoJcZ9HaRGyWh2gm8M,5893
|
|
59
|
+
anemoi/datasets/create/functions/sources/xarray/field.py,sha256=bUTgod2XuV3OGDaK7kcGiUqC5oQ6tuCWbHb67fo9lsY,3186
|
|
60
|
+
anemoi/datasets/create/functions/sources/xarray/fieldlist.py,sha256=B0toXCiYFBFF0fvxpy5bxnan2dfptWMl51_JO8TrH3A,5557
|
|
61
|
+
anemoi/datasets/create/functions/sources/xarray/flavour.py,sha256=_tVSrvDPJ5pXAJhmha3UA1a0Gr2FrPhcjRFoAMtX-EU,10538
|
|
55
62
|
anemoi/datasets/create/functions/sources/xarray/grid.py,sha256=KSlhXj5dHeRQVeZd26CChMyC1iqYsH5XqxPC2VA_YJc,1169
|
|
56
|
-
anemoi/datasets/create/functions/sources/xarray/metadata.py,sha256=
|
|
57
|
-
anemoi/datasets/create/functions/sources/xarray/time.py,sha256=
|
|
58
|
-
anemoi/datasets/create/functions/sources/xarray/variable.py,sha256=
|
|
59
|
-
anemoi/datasets/create/statistics/__init__.py,sha256=
|
|
63
|
+
anemoi/datasets/create/functions/sources/xarray/metadata.py,sha256=_HQgAY5qI39M-5JhEx2q3APgqqmXE0Ed3GqoUvttNuE,4721
|
|
64
|
+
anemoi/datasets/create/functions/sources/xarray/time.py,sha256=WGO8m9J7fF-GjgBEJ0Dmqp-M2-vMsrkH-aSG0QBOVYU,4726
|
|
65
|
+
anemoi/datasets/create/functions/sources/xarray/variable.py,sha256=THd7uterDPJlpypRCCgTwKaaGjkV74s6WmuveH6qyBE,4474
|
|
66
|
+
anemoi/datasets/create/statistics/__init__.py,sha256=wMCncdfpLXxfDA9-RMtI2raVGP6dGqXfSolCXEzzITE,12704
|
|
60
67
|
anemoi/datasets/create/statistics/summary.py,sha256=sgmhA24y3VRyjmDUgTnPIqcHSlWBbFA0qynx6gJ9Xw8,3370
|
|
61
|
-
anemoi/datasets/data/__init__.py,sha256=
|
|
62
|
-
anemoi/datasets/data/concat.py,sha256=
|
|
63
|
-
anemoi/datasets/data/dataset.py,sha256=
|
|
68
|
+
anemoi/datasets/data/__init__.py,sha256=usTH1SLkeveBbYkbBTc7rP4G3mCGNa8A74vPjXjlOSw,1067
|
|
69
|
+
anemoi/datasets/data/concat.py,sha256=SYASNNngnAyz3rC4ENzrGAu91rnhd6uwP0kGX9aMZAQ,5269
|
|
70
|
+
anemoi/datasets/data/dataset.py,sha256=u1RKLEJer6gY6n4odBHKK4CEozWCUQjFJMFqCCF_FPo,10044
|
|
64
71
|
anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
|
|
65
72
|
anemoi/datasets/data/debug.py,sha256=PcyrjgxaLzeb_vf12pvUtPPVvBRHNm1SimythZvqsP4,6303
|
|
66
73
|
anemoi/datasets/data/ensemble.py,sha256=AsP7Xx0ZHLoZs6a4EC0jtyGYIcOvZvvKXhgNsIvqIN8,1137
|
|
67
|
-
anemoi/datasets/data/forwards.py,sha256=
|
|
68
|
-
anemoi/datasets/data/grids.py,sha256=
|
|
74
|
+
anemoi/datasets/data/forwards.py,sha256=SfB3o8Y-nQEOkS5gI_6wtipvzB5peF_YkwW9cOMhLMg,8167
|
|
75
|
+
anemoi/datasets/data/grids.py,sha256=KiHpN1Ne0DnQM42buBy0sIAZHNaC2kuEFwkk6ohIooU,8046
|
|
69
76
|
anemoi/datasets/data/indexing.py,sha256=625m__JG5m_tDMrkz1hB6Vydenwt0oHuyAlc-o3Zwos,4799
|
|
77
|
+
anemoi/datasets/data/interpolate.py,sha256=usnYgjoDPfnUNg_lZl3KsFTsJ7kOsf7YaHwarsBU7Ag,4163
|
|
70
78
|
anemoi/datasets/data/join.py,sha256=dtCBbMTicqrRPxfBULi3RwEcQBLhQpIcvCjdN5A3XUU,4892
|
|
71
79
|
anemoi/datasets/data/masked.py,sha256=czAv1ZfZ9q6Wr4RqI2Xj8SEm7yoCgJrwMl-CPDs_wSI,3857
|
|
72
|
-
anemoi/datasets/data/misc.py,sha256=
|
|
73
|
-
anemoi/datasets/data/
|
|
80
|
+
anemoi/datasets/data/misc.py,sha256=8ZaLhgHmUmP77X0_6scp36BJJzz544nHVQRAamW6x-Y,9821
|
|
81
|
+
anemoi/datasets/data/missing.py,sha256=OyW8cRow1v641Vgv-IY5BnUxH2Ob5P2QjgrVR1Szjl0,7157
|
|
82
|
+
anemoi/datasets/data/select.py,sha256=tProUPIiRwSe_H9cwwn_4iMX1uZEIQ7q5KsegBgSmdc,3971
|
|
74
83
|
anemoi/datasets/data/statistics.py,sha256=lZCcKw9s7ttMBEp6ANyxtbXoZZvchhE7SClq-D4AUR8,1645
|
|
75
|
-
anemoi/datasets/data/stores.py,sha256=
|
|
76
|
-
anemoi/datasets/data/subset.py,sha256=
|
|
77
|
-
anemoi/datasets/data/unchecked.py,sha256=
|
|
78
|
-
anemoi/datasets/
|
|
79
|
-
anemoi/datasets/dates/
|
|
84
|
+
anemoi/datasets/data/stores.py,sha256=4F2tCsG3p_ZI0Ut2FdtfPCGClP8U9U6V_12l8-aIGbg,11962
|
|
85
|
+
anemoi/datasets/data/subset.py,sha256=peyHlIvno0uUzqW0HS9zjqThmyXn6rpvVWh-925XHnE,4692
|
|
86
|
+
anemoi/datasets/data/unchecked.py,sha256=xLOCU-O3OpfAi3bd-XZEpfDTnZfFGqdhh55d8D_u3wQ,4184
|
|
87
|
+
anemoi/datasets/data/xy.py,sha256=_k1z_2INvA0v7zsm5lxfVvf7ymBFu6Tr9PxvtdB9kFg,3002
|
|
88
|
+
anemoi/datasets/dates/__init__.py,sha256=odF5RnQtTRozEys4_17bR20jWzUGspFCDM0i7UC6uVs,5087
|
|
89
|
+
anemoi/datasets/dates/groups.py,sha256=lBnrutmQ7WNv-Asyl7eI95Uhi0zfT_3Kb0R_Xm1UNnM,3469
|
|
80
90
|
anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
81
|
-
anemoi_datasets-0.4.
|
|
82
|
-
anemoi_datasets-0.4.
|
|
83
|
-
anemoi_datasets-0.4.
|
|
84
|
-
anemoi_datasets-0.4.
|
|
85
|
-
anemoi_datasets-0.4.
|
|
86
|
-
anemoi_datasets-0.4.
|
|
91
|
+
anemoi_datasets-0.4.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
92
|
+
anemoi_datasets-0.4.5.dist-info/METADATA,sha256=4jEPNynR2lfXdnhXNlzCBb8jtEgSQ69n8WSTtaxrm9Q,16707
|
|
93
|
+
anemoi_datasets-0.4.5.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
|
|
94
|
+
anemoi_datasets-0.4.5.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
|
|
95
|
+
anemoi_datasets-0.4.5.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
|
|
96
|
+
anemoi_datasets-0.4.5.dist-info/RECORD,,
|