anemoi-utils 0.4.9__py3-none-any.whl → 0.4.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of anemoi-utils might be problematic. Click here for more details.
- anemoi/utils/_version.py +2 -2
- anemoi/utils/caching.py +85 -35
- anemoi/utils/commands/requests.py +44 -0
- anemoi/utils/config.py +47 -4
- anemoi/utils/devtools.py +83 -0
- anemoi/utils/grids.py +97 -0
- anemoi/utils/humanize.py +33 -0
- anemoi/utils/logs.py +40 -0
- anemoi/utils/mars/requests.py +22 -0
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/METADATA +21 -21
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/RECORD +15 -10
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/WHEEL +1 -1
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/LICENSE +0 -0
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/entry_points.txt +0 -0
- {anemoi_utils-0.4.9.dist-info → anemoi_utils-0.4.11.dist-info}/top_level.txt +0 -0
anemoi/utils/_version.py
CHANGED
anemoi/utils/caching.py
CHANGED
|
@@ -14,47 +14,31 @@ import os
|
|
|
14
14
|
import time
|
|
15
15
|
from threading import Lock
|
|
16
16
|
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
17
19
|
LOCK = Lock()
|
|
18
20
|
CACHE = {}
|
|
19
21
|
|
|
20
22
|
|
|
21
|
-
def
|
|
22
|
-
|
|
23
|
-
key = json.dumps(key, sort_keys=True)
|
|
24
|
-
m = hashlib.md5()
|
|
25
|
-
m.update(key.encode("utf-8"))
|
|
26
|
-
m = m.hexdigest()
|
|
27
|
-
|
|
28
|
-
if m in CACHE:
|
|
29
|
-
return CACHE[m]
|
|
30
|
-
|
|
31
|
-
path = os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
|
|
32
|
-
os.makedirs(path, exist_ok=True)
|
|
33
|
-
|
|
34
|
-
filename = os.path.join(path, m)
|
|
35
|
-
if os.path.exists(filename):
|
|
36
|
-
with open(filename, "r") as f:
|
|
37
|
-
data = json.load(f)
|
|
38
|
-
if expires is None or data["expires"] > time.time():
|
|
39
|
-
if data["key"] == key:
|
|
40
|
-
return data["value"]
|
|
41
|
-
|
|
42
|
-
value = proc()
|
|
43
|
-
data = {"key": key, "value": value}
|
|
44
|
-
if expires is not None:
|
|
45
|
-
data["expires"] = time.time() + expires
|
|
23
|
+
def _get_cache_path(collection):
|
|
24
|
+
return os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
|
|
46
25
|
|
|
47
|
-
with open(filename, "w") as f:
|
|
48
|
-
json.dump(data, f)
|
|
49
26
|
|
|
50
|
-
|
|
51
|
-
|
|
27
|
+
def clean_cache(collection="default"):
|
|
28
|
+
global CACHE
|
|
29
|
+
CACHE = {}
|
|
30
|
+
path = _get_cache_path(collection)
|
|
31
|
+
if not os.path.exists(path):
|
|
32
|
+
return
|
|
33
|
+
for filename in os.listdir(path):
|
|
34
|
+
os.remove(os.path.join(path, filename))
|
|
52
35
|
|
|
53
36
|
|
|
54
|
-
class
|
|
55
|
-
"""
|
|
37
|
+
class Cacher:
|
|
38
|
+
"""This class implements a simple caching mechanism.
|
|
39
|
+
Private class, do not use directly"""
|
|
56
40
|
|
|
57
|
-
def __init__(self, collection
|
|
41
|
+
def __init__(self, collection, expires):
|
|
58
42
|
self.collection = collection
|
|
59
43
|
self.expires = expires
|
|
60
44
|
|
|
@@ -64,11 +48,77 @@ class cached:
|
|
|
64
48
|
|
|
65
49
|
def wrapped(*args, **kwargs):
|
|
66
50
|
with LOCK:
|
|
67
|
-
return cache(
|
|
51
|
+
return self.cache(
|
|
68
52
|
(full, args, kwargs),
|
|
69
53
|
lambda: func(*args, **kwargs),
|
|
70
|
-
self.collection,
|
|
71
|
-
self.expires,
|
|
72
54
|
)
|
|
73
55
|
|
|
74
56
|
return wrapped
|
|
57
|
+
|
|
58
|
+
def cache(self, key, proc):
|
|
59
|
+
|
|
60
|
+
key = json.dumps(key, sort_keys=True)
|
|
61
|
+
m = hashlib.md5()
|
|
62
|
+
m.update(key.encode("utf-8"))
|
|
63
|
+
m = m.hexdigest()
|
|
64
|
+
|
|
65
|
+
if m in CACHE:
|
|
66
|
+
return CACHE[m]
|
|
67
|
+
|
|
68
|
+
path = _get_cache_path(self.collection)
|
|
69
|
+
|
|
70
|
+
filename = os.path.join(path, m) + self.ext
|
|
71
|
+
if os.path.exists(filename):
|
|
72
|
+
data = self.load(filename)
|
|
73
|
+
if self.expires is None or data["expires"] > time.time():
|
|
74
|
+
if data["key"] == key:
|
|
75
|
+
return data["value"]
|
|
76
|
+
|
|
77
|
+
value = proc()
|
|
78
|
+
data = {"key": key, "value": value}
|
|
79
|
+
if self.expires is not None:
|
|
80
|
+
data["expires"] = time.time() + self.expires
|
|
81
|
+
|
|
82
|
+
os.makedirs(path, exist_ok=True)
|
|
83
|
+
temp_filename = self.save(filename, data)
|
|
84
|
+
os.rename(temp_filename, filename)
|
|
85
|
+
|
|
86
|
+
CACHE[m] = value
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class JsonCacher(Cacher):
|
|
91
|
+
ext = ""
|
|
92
|
+
|
|
93
|
+
def save(self, path, data):
|
|
94
|
+
temp_path = path + ".tmp"
|
|
95
|
+
with open(temp_path, "w") as f:
|
|
96
|
+
json.dump(data, f)
|
|
97
|
+
return temp_path
|
|
98
|
+
|
|
99
|
+
def load(self, path):
|
|
100
|
+
with open(path, "r") as f:
|
|
101
|
+
return json.load(f)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class NpzCacher(Cacher):
|
|
105
|
+
ext = ".npz"
|
|
106
|
+
|
|
107
|
+
def save(self, path, data):
|
|
108
|
+
temp_path = path + ".tmp.npz"
|
|
109
|
+
np.savez(temp_path, **data)
|
|
110
|
+
return temp_path
|
|
111
|
+
|
|
112
|
+
def load(self, path):
|
|
113
|
+
return np.load(path, allow_pickle=True)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# PUBLIC API
|
|
117
|
+
def cached(collection="default", expires=None, encoding="json"):
|
|
118
|
+
"""Decorator to cache the result of a function.
|
|
119
|
+
|
|
120
|
+
Default is to use a json file to store the cache, but you can also use npz files
|
|
121
|
+
to cache dict of numpy arrays.
|
|
122
|
+
|
|
123
|
+
"""
|
|
124
|
+
return dict(json=JsonCacher, npz=NpzCacher)[encoding](collection, expires)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
|
|
2
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
3
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
4
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
5
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
6
|
+
# nor does it submit to any jurisdiction.
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
|
|
10
|
+
from anemoi.utils.mars.requests import print_request
|
|
11
|
+
|
|
12
|
+
from . import Command
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Requests(Command):
|
|
16
|
+
"""Convert a JSON requests file to MARS format."""
|
|
17
|
+
|
|
18
|
+
def add_arguments(self, command_parser):
|
|
19
|
+
command_parser.add_argument("input")
|
|
20
|
+
command_parser.add_argument("output")
|
|
21
|
+
command_parser.add_argument("--verb", default="retrieve")
|
|
22
|
+
command_parser.add_argument("--only-one-field", action="store_true")
|
|
23
|
+
|
|
24
|
+
def run(self, args):
|
|
25
|
+
with open(args.input) as f:
|
|
26
|
+
requests = json.load(f)
|
|
27
|
+
|
|
28
|
+
if args.only_one_field:
|
|
29
|
+
for r in requests:
|
|
30
|
+
for key in (
|
|
31
|
+
"grid",
|
|
32
|
+
"area",
|
|
33
|
+
):
|
|
34
|
+
r.pop(key, None)
|
|
35
|
+
for k, v in list(r.items()):
|
|
36
|
+
if isinstance(v, list):
|
|
37
|
+
r[k] = v[-1]
|
|
38
|
+
|
|
39
|
+
with open(args.output, "w") as f:
|
|
40
|
+
for r in requests:
|
|
41
|
+
print_request(args.verb, r, file=f)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
command = Requests
|
anemoi/utils/config.py
CHANGED
|
@@ -50,14 +50,14 @@ class DotDict(dict):
|
|
|
50
50
|
super().__init__(*args, **kwargs)
|
|
51
51
|
|
|
52
52
|
for k, v in self.items():
|
|
53
|
-
if isinstance(v, dict):
|
|
53
|
+
if isinstance(v, dict) or is_omegaconf_dict(v):
|
|
54
54
|
self[k] = DotDict(v)
|
|
55
55
|
|
|
56
|
-
if isinstance(v, list):
|
|
57
|
-
self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
|
|
56
|
+
if isinstance(v, list) or is_omegaconf_list(v):
|
|
57
|
+
self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
|
|
58
58
|
|
|
59
59
|
if isinstance(v, tuple):
|
|
60
|
-
self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
|
|
60
|
+
self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
|
|
61
61
|
|
|
62
62
|
@classmethod
|
|
63
63
|
def from_file(cls, path: str):
|
|
@@ -106,6 +106,24 @@ class DotDict(dict):
|
|
|
106
106
|
return f"DotDict({super().__repr__()})"
|
|
107
107
|
|
|
108
108
|
|
|
109
|
+
def is_omegaconf_dict(value) -> bool:
|
|
110
|
+
try:
|
|
111
|
+
from omegaconf import DictConfig
|
|
112
|
+
|
|
113
|
+
return isinstance(value, DictConfig)
|
|
114
|
+
except ImportError:
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def is_omegaconf_list(value) -> bool:
|
|
119
|
+
try:
|
|
120
|
+
from omegaconf import ListConfig
|
|
121
|
+
|
|
122
|
+
return isinstance(value, ListConfig)
|
|
123
|
+
except ImportError:
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
|
|
109
127
|
CONFIG = {}
|
|
110
128
|
CHECKED = {}
|
|
111
129
|
CONFIG_LOCK = threading.RLock()
|
|
@@ -205,6 +223,23 @@ def load_any_dict_format(path) -> dict:
|
|
|
205
223
|
if path.endswith(".toml"):
|
|
206
224
|
with open(path, "rb") as f:
|
|
207
225
|
return tomllib.load(f)
|
|
226
|
+
|
|
227
|
+
if path == "-":
|
|
228
|
+
import sys
|
|
229
|
+
|
|
230
|
+
config = sys.stdin.read()
|
|
231
|
+
|
|
232
|
+
parsers = [(yaml.safe_load, "yaml"), (json.loads, "json"), (tomllib.loads, "toml")]
|
|
233
|
+
|
|
234
|
+
for parser, parser_type in parsers:
|
|
235
|
+
try:
|
|
236
|
+
LOG.debug(f"Trying {parser_type} parser for stdin")
|
|
237
|
+
return parser(config)
|
|
238
|
+
except Exception:
|
|
239
|
+
pass
|
|
240
|
+
|
|
241
|
+
raise ValueError("Failed to parse configuration from stdin")
|
|
242
|
+
|
|
208
243
|
except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError) as e:
|
|
209
244
|
LOG.warning(f"Failed to parse config file {path}", exc_info=e)
|
|
210
245
|
raise ValueError(f"Failed to parse config file {path} [{e}]")
|
|
@@ -376,3 +411,11 @@ def find(metadata, what, result=None, *, select: callable = None):
|
|
|
376
411
|
find(v, what, result)
|
|
377
412
|
|
|
378
413
|
return result
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def merge_configs(*configs):
|
|
417
|
+
result = {}
|
|
418
|
+
for config in configs:
|
|
419
|
+
_merge_dicts(result, config)
|
|
420
|
+
|
|
421
|
+
return result
|
anemoi/utils/devtools.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
|
+
#
|
|
3
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
import cartopy.crs as ccrs
|
|
12
|
+
import cartopy.feature as cfeature
|
|
13
|
+
import matplotlib.pyplot as plt
|
|
14
|
+
import matplotlib.tri as tri
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
"""FOR DEVELOPMENT PURPOSES ONLY
|
|
18
|
+
|
|
19
|
+
This module contains
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
# TODO: use earthkit-plots
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def fix(lons):
|
|
27
|
+
return np.where(lons > 180, lons - 360, lons)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def plot_values(
|
|
31
|
+
values, latitudes, longitudes, title=None, missing_value=None, min_value=None, max_value=None, **kwargs
|
|
32
|
+
):
|
|
33
|
+
|
|
34
|
+
_, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()})
|
|
35
|
+
ax.coastlines()
|
|
36
|
+
ax.add_feature(cfeature.BORDERS, linestyle=":")
|
|
37
|
+
|
|
38
|
+
missing_values = np.isnan(values)
|
|
39
|
+
|
|
40
|
+
if missing_value is None:
|
|
41
|
+
values = values[~missing_values]
|
|
42
|
+
longitudes = longitudes[~missing_values]
|
|
43
|
+
latitudes = latitudes[~missing_values]
|
|
44
|
+
else:
|
|
45
|
+
values = np.where(missing_values, missing_value, values)
|
|
46
|
+
|
|
47
|
+
if max_value is not None:
|
|
48
|
+
values = np.where(values > max_value, max_value, values)
|
|
49
|
+
|
|
50
|
+
if min_value is not None:
|
|
51
|
+
values = np.where(values < min_value, min_value, values)
|
|
52
|
+
|
|
53
|
+
triangulation = tri.Triangulation(fix(longitudes), latitudes)
|
|
54
|
+
|
|
55
|
+
levels = kwargs.pop("levels", 10)
|
|
56
|
+
|
|
57
|
+
_ = ax.tricontourf(triangulation, values, levels=levels, transform=ccrs.PlateCarree())
|
|
58
|
+
|
|
59
|
+
options = dict(
|
|
60
|
+
levels=levels,
|
|
61
|
+
colors="black",
|
|
62
|
+
linewidths=0.5,
|
|
63
|
+
transform=ccrs.PlateCarree(),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
options.update(kwargs)
|
|
67
|
+
|
|
68
|
+
ax.tricontour(
|
|
69
|
+
triangulation,
|
|
70
|
+
values,
|
|
71
|
+
**options,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
if title is not None:
|
|
75
|
+
ax.set_title(title)
|
|
76
|
+
|
|
77
|
+
return ax
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def plot_field(field, title=None, **kwargs):
|
|
81
|
+
values = field.to_numpy(flatten=True)
|
|
82
|
+
latitudes, longitudes = field.grid_points()
|
|
83
|
+
return plot_values(values, latitudes, longitudes, title=title, **kwargs)
|
anemoi/utils/grids.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# (C) Copyright 2025 Anemoi contributors.
|
|
2
|
+
#
|
|
3
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
"""Utilities for working with grids.
|
|
12
|
+
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
from io import BytesIO
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import requests
|
|
21
|
+
|
|
22
|
+
from .caching import cached
|
|
23
|
+
|
|
24
|
+
LOG = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def xyz_to_latlon(x, y, z):
|
|
31
|
+
return (
|
|
32
|
+
np.rad2deg(np.arcsin(np.minimum(1.0, np.maximum(-1.0, z)))),
|
|
33
|
+
np.rad2deg(np.arctan2(y, x)),
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def latlon_to_xyz(lat, lon, radius=1.0):
|
|
38
|
+
# https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
|
|
39
|
+
# We assume that the Earth is a sphere of radius 1 so N(phi) = 1
|
|
40
|
+
# We assume h = 0
|
|
41
|
+
#
|
|
42
|
+
phi = np.deg2rad(lat)
|
|
43
|
+
lda = np.deg2rad(lon)
|
|
44
|
+
|
|
45
|
+
cos_phi = np.cos(phi)
|
|
46
|
+
cos_lda = np.cos(lda)
|
|
47
|
+
sin_phi = np.sin(phi)
|
|
48
|
+
sin_lda = np.sin(lda)
|
|
49
|
+
|
|
50
|
+
x = cos_phi * cos_lda * radius
|
|
51
|
+
y = cos_phi * sin_lda * radius
|
|
52
|
+
z = sin_phi * radius
|
|
53
|
+
|
|
54
|
+
return x, y, z
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def nearest_grid_points(source_latitudes, source_longitudes, target_latitudes, target_longitudes):
|
|
58
|
+
from scipy.spatial import cKDTree
|
|
59
|
+
|
|
60
|
+
source_xyz = latlon_to_xyz(source_latitudes, source_longitudes)
|
|
61
|
+
source_points = np.array(source_xyz).transpose()
|
|
62
|
+
|
|
63
|
+
target_xyz = latlon_to_xyz(target_latitudes, target_longitudes)
|
|
64
|
+
target_points = np.array(target_xyz).transpose()
|
|
65
|
+
|
|
66
|
+
_, indices = cKDTree(source_points).query(target_points, k=1)
|
|
67
|
+
return indices
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@cached(collection="grids", encoding="npz")
|
|
71
|
+
def _grids(name):
|
|
72
|
+
from anemoi.utils.config import load_config
|
|
73
|
+
|
|
74
|
+
user_path = load_config().get("utils", {}).get("grids_path")
|
|
75
|
+
if user_path:
|
|
76
|
+
path = os.path.expanduser(os.path.join(user_path, f"grid-{name}.npz"))
|
|
77
|
+
if os.path.exists(path):
|
|
78
|
+
LOG.warning("Loading grids from custom user path %s", path)
|
|
79
|
+
with open(path, "rb") as f:
|
|
80
|
+
return f.read()
|
|
81
|
+
else:
|
|
82
|
+
LOG.warning("Custom user path %s does not exist", path)
|
|
83
|
+
|
|
84
|
+
url = GRIDS_URL_PATTERN.format(name=name.lower())
|
|
85
|
+
LOG.warning("Downloading grids from %s", url)
|
|
86
|
+
response = requests.get(url)
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
return response.content
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def grids(name):
|
|
92
|
+
if name.endswith(".npz"):
|
|
93
|
+
return dict(np.load(name))
|
|
94
|
+
|
|
95
|
+
data = _grids(name)
|
|
96
|
+
npz = np.load(BytesIO(data))
|
|
97
|
+
return dict(npz)
|
anemoi/utils/humanize.py
CHANGED
|
@@ -689,3 +689,36 @@ def print_dates(dates) -> None:
|
|
|
689
689
|
A list of dates, as datetime objects or strings.
|
|
690
690
|
"""
|
|
691
691
|
print(compress_dates(dates))
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def make_list_int(value) -> list:
|
|
695
|
+
"""Convert a string like "1/2/3" or "1/to/3" or "1/to/10/by/2" to a list of integers.
|
|
696
|
+
|
|
697
|
+
Parameters
|
|
698
|
+
----------
|
|
699
|
+
value : str, list, tuple, int
|
|
700
|
+
The value to convert to a list of integers.
|
|
701
|
+
|
|
702
|
+
Returns
|
|
703
|
+
-------
|
|
704
|
+
list
|
|
705
|
+
A list of integers.
|
|
706
|
+
"""
|
|
707
|
+
if isinstance(value, str):
|
|
708
|
+
if "/" not in value:
|
|
709
|
+
return [int(value)]
|
|
710
|
+
bits = value.split("/")
|
|
711
|
+
if len(bits) == 3 and bits[1].lower() == "to":
|
|
712
|
+
value = list(range(int(bits[0]), int(bits[2]) + 1, 1))
|
|
713
|
+
|
|
714
|
+
elif len(bits) == 5 and bits[1].lower() == "to" and bits[3].lower() == "by":
|
|
715
|
+
value = list(range(int(bits[0]), int(bits[2]) + int(bits[4]), int(bits[4])))
|
|
716
|
+
|
|
717
|
+
if isinstance(value, list):
|
|
718
|
+
return value
|
|
719
|
+
if isinstance(value, tuple):
|
|
720
|
+
return value
|
|
721
|
+
if isinstance(value, int):
|
|
722
|
+
return [value]
|
|
723
|
+
|
|
724
|
+
raise ValueError(f"Cannot make list from {value}")
|
anemoi/utils/logs.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
|
+
#
|
|
3
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
"""Logging utilities."""
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import threading
|
|
15
|
+
|
|
16
|
+
thread_local = threading.local()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
LOGGER = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def set_logging_name(name):
|
|
23
|
+
thread_local.logging_name = name
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ThreadCustomFormatter(logging.Formatter):
|
|
27
|
+
def format(self, record):
|
|
28
|
+
record.logging_name = thread_local.logging_name
|
|
29
|
+
return super().format(record)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def enable_logging_name(name="main"):
|
|
33
|
+
thread_local.logging_name = name
|
|
34
|
+
|
|
35
|
+
formatter = ThreadCustomFormatter("%(asctime)s - %(logging_name)s - %(levelname)s - %(message)s")
|
|
36
|
+
|
|
37
|
+
logger = logging.getLogger()
|
|
38
|
+
|
|
39
|
+
for handler in logger.handlers:
|
|
40
|
+
handler.setFormatter(formatter)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
|
|
2
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
3
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
4
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
5
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
6
|
+
# nor does it submit to any jurisdiction.
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def print_request(verb, request, file=sys.stdout):
|
|
12
|
+
r = [verb]
|
|
13
|
+
for k, v in request.items():
|
|
14
|
+
if not isinstance(v, (list, tuple, set)):
|
|
15
|
+
v = [v]
|
|
16
|
+
v = [str(_) for _ in v]
|
|
17
|
+
v = "/".join(v)
|
|
18
|
+
r.append(f"{k}={v}")
|
|
19
|
+
|
|
20
|
+
r = ",\n ".join(r)
|
|
21
|
+
print(r, file=file)
|
|
22
|
+
print(file=file)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: anemoi-utils
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.11
|
|
4
4
|
Summary: A package to hold various functions to support training of ML models on ECMWF data.
|
|
5
5
|
Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
|
|
6
|
-
License:
|
|
6
|
+
License: Apache License
|
|
7
7
|
Version 2.0, January 2004
|
|
8
8
|
http://www.apache.org/licenses/
|
|
9
9
|
|
|
@@ -225,31 +225,31 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
|
225
225
|
Requires-Python: >=3.9
|
|
226
226
|
License-File: LICENSE
|
|
227
227
|
Requires-Dist: aniso8601
|
|
228
|
+
Requires-Dist: importlib-metadata; python_version < "3.10"
|
|
229
|
+
Requires-Dist: numpy
|
|
228
230
|
Requires-Dist: python-dateutil
|
|
229
231
|
Requires-Dist: pyyaml
|
|
232
|
+
Requires-Dist: tomli; python_version < "3.11"
|
|
230
233
|
Requires-Dist: tqdm
|
|
231
|
-
Requires-Dist: importlib-metadata ; python_version < "3.10"
|
|
232
|
-
Requires-Dist: tomli ; python_version < "3.11"
|
|
233
234
|
Provides-Extra: all
|
|
234
|
-
Requires-Dist: anemoi-utils[grib,provenance,text]
|
|
235
|
+
Requires-Dist: anemoi-utils[grib,provenance,text]; extra == "all"
|
|
235
236
|
Provides-Extra: dev
|
|
236
|
-
Requires-Dist: anemoi-utils[all,docs,tests]
|
|
237
|
+
Requires-Dist: anemoi-utils[all,docs,tests]; extra == "dev"
|
|
237
238
|
Provides-Extra: docs
|
|
238
|
-
Requires-Dist: nbsphinx
|
|
239
|
-
Requires-Dist: pandoc
|
|
240
|
-
Requires-Dist: requests
|
|
241
|
-
Requires-Dist: sphinx
|
|
242
|
-
Requires-Dist: sphinx-argparse
|
|
243
|
-
Requires-Dist: sphinx-rtd-theme
|
|
244
|
-
Requires-Dist: termcolor
|
|
239
|
+
Requires-Dist: nbsphinx; extra == "docs"
|
|
240
|
+
Requires-Dist: pandoc; extra == "docs"
|
|
241
|
+
Requires-Dist: requests; extra == "docs"
|
|
242
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
243
|
+
Requires-Dist: sphinx-argparse<0.5; extra == "docs"
|
|
244
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
245
|
+
Requires-Dist: termcolor; extra == "docs"
|
|
245
246
|
Provides-Extra: grib
|
|
246
|
-
Requires-Dist: requests
|
|
247
|
+
Requires-Dist: requests; extra == "grib"
|
|
247
248
|
Provides-Extra: provenance
|
|
248
|
-
Requires-Dist: gitpython
|
|
249
|
-
Requires-Dist: nvsmi
|
|
249
|
+
Requires-Dist: gitpython; extra == "provenance"
|
|
250
|
+
Requires-Dist: nvsmi; extra == "provenance"
|
|
250
251
|
Provides-Extra: tests
|
|
251
|
-
Requires-Dist: pytest
|
|
252
|
+
Requires-Dist: pytest; extra == "tests"
|
|
252
253
|
Provides-Extra: text
|
|
253
|
-
Requires-Dist: termcolor
|
|
254
|
-
Requires-Dist: wcwidth
|
|
255
|
-
|
|
254
|
+
Requires-Dist: termcolor; extra == "text"
|
|
255
|
+
Requires-Dist: wcwidth; extra == "text"
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
anemoi/utils/__init__.py,sha256=0u0eIdu5-H1frf6V4KHpNmlh_SS-bJnxjzIejlsLqdw,702
|
|
2
2
|
anemoi/utils/__main__.py,sha256=5NW2A3OgTimB4ptwYThivIRSeCrvabMuvnr8mmnVx0E,715
|
|
3
|
-
anemoi/utils/_version.py,sha256=
|
|
4
|
-
anemoi/utils/caching.py,sha256=
|
|
3
|
+
anemoi/utils/_version.py,sha256=XaB1qSGFgL3t6k654LeY_W4E2hovj_ZK7X1yDkanD5E,413
|
|
4
|
+
anemoi/utils/caching.py,sha256=UkHQOKcBoB6xni83qgTEGfkyam7iqu1YiuBLZQIn9RM,3208
|
|
5
5
|
anemoi/utils/checkpoints.py,sha256=q8QqKlZ6qChjzEfq7KM1gVXuyqgsVRGIb4dJFtkGk58,7774
|
|
6
6
|
anemoi/utils/cli.py,sha256=rmMP60VY3em99rQP6TCrKibMngWwVe5h_0GDcf16c5U,4117
|
|
7
7
|
anemoi/utils/compatibility.py,sha256=0_nIcbdQbNMrS6AkqrBgJGJlSJXW8R23ncaZaDwdJ4c,2190
|
|
8
|
-
anemoi/utils/config.py,sha256=
|
|
8
|
+
anemoi/utils/config.py,sha256=ADCksBl6qrJpU9eebVebwkthNanzgCOxBgzlqCUTwbg,10962
|
|
9
9
|
anemoi/utils/dates.py,sha256=wwYD5_QI7EWY_jhpENNYtL5O7fjwYkzmqHkNoayvmrY,12452
|
|
10
|
+
anemoi/utils/devtools.py,sha256=Mns5vU9o2HrO4zS1e0-W4gBIhk8xHrhcB7wLR_q6OiA,2172
|
|
10
11
|
anemoi/utils/grib.py,sha256=zBICyOsYtR_9px1C5UDT6wL_D6kiIhUi_00kjFmas5c,3047
|
|
12
|
+
anemoi/utils/grids.py,sha256=tqhH8ZiRS9Re7xQZHVKtl8bBtqH_kDVOGHfDTxu3RuI,2708
|
|
11
13
|
anemoi/utils/hindcasts.py,sha256=TEYDmrZUajuhp_dfWeg6z5c6XfntE-mwugUQJyAgUco,1419
|
|
12
|
-
anemoi/utils/humanize.py,sha256=
|
|
14
|
+
anemoi/utils/humanize.py,sha256=ZD5UMD7m79I1h_IoIQFnd4FZR5K9VARw9cVaNrD1QdM,17579
|
|
15
|
+
anemoi/utils/logs.py,sha256=o0xXiO2BdG_bZkljxxI2TKlCiA5QbWHgAUlYM53lirE,1058
|
|
13
16
|
anemoi/utils/provenance.py,sha256=SqOiNoY1y36Zec83Pjt7OhihbwxMyknscfmogHCuriA,10894
|
|
14
17
|
anemoi/utils/registry.py,sha256=Iit_CfTGuoVffXkZA2A5mUXb4AdGIUX9TpnUqWT4HJ0,4291
|
|
15
18
|
anemoi/utils/s3.py,sha256=UOEETko08hnIXeFy8u10eQbqpcape9d-L6IgsjFMe18,2473
|
|
@@ -19,14 +22,16 @@ anemoi/utils/text.py,sha256=Xfr_3wvsjg7m-BwvdJVz1bV6f5KNMnGIIFRtXaiMfbs,10496
|
|
|
19
22
|
anemoi/utils/timer.py,sha256=Twnr3GZu-n0WzgboELRKJWs87qyDYqy6Dwr9cQ_JG18,1803
|
|
20
23
|
anemoi/utils/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
|
|
21
24
|
anemoi/utils/commands/config.py,sha256=zt4PFATYJ-zs0C5mpUlrQ4Fj5m1kM3CcsszUP1VBbzA,816
|
|
25
|
+
anemoi/utils/commands/requests.py,sha256=7joRYnJUzJh5O8Pqkqa-s9M9woHy-Z86czp00uCZXGc,1448
|
|
22
26
|
anemoi/utils/mars/__init__.py,sha256=kvbu-gSaYI9jSNEzfQltrtHPVIameYGoLjOJKwI7x_U,1723
|
|
23
27
|
anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
|
|
28
|
+
anemoi/utils/mars/requests.py,sha256=0khe_mbq4GNueR_B8fGPTBoWHtCfjQvtoKXOSVm6La4,759
|
|
24
29
|
anemoi/utils/remote/__init__.py,sha256=-_AA1xm9GpagW5zP0PGpz-3SRKEUjw_AGSNd_bhuh7g,11639
|
|
25
30
|
anemoi/utils/remote/s3.py,sha256=hykbVlh1_aFI00FWjgm_FWIMfVCTFiQf_cq8_gAo31s,11976
|
|
26
31
|
anemoi/utils/remote/ssh.py,sha256=3lqFpY9CEor_DvIK9ZxSmj3rND-366Sm9R3Vw61sWSs,4695
|
|
27
|
-
anemoi_utils-0.4.
|
|
28
|
-
anemoi_utils-0.4.
|
|
29
|
-
anemoi_utils-0.4.
|
|
30
|
-
anemoi_utils-0.4.
|
|
31
|
-
anemoi_utils-0.4.
|
|
32
|
-
anemoi_utils-0.4.
|
|
32
|
+
anemoi_utils-0.4.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
33
|
+
anemoi_utils-0.4.11.dist-info/METADATA,sha256=YYQUqE12vxm1wx4kNf-ey148QJGs0Ik823cWX8tufRM,15258
|
|
34
|
+
anemoi_utils-0.4.11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
35
|
+
anemoi_utils-0.4.11.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
|
|
36
|
+
anemoi_utils-0.4.11.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
|
|
37
|
+
anemoi_utils-0.4.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|