anemoi-utils 0.4.23__py3-none-any.whl → 0.4.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of anemoi-utils might be problematic. Click here for more details.
- anemoi/utils/_version.py +2 -2
- anemoi/utils/cli.py +14 -4
- anemoi/utils/commands/transfer.py +74 -0
- anemoi/utils/grids.py +27 -0
- anemoi/utils/remote/__init__.py +8 -3
- anemoi/utils/remote/s3.py +107 -26
- anemoi/utils/testing.py +101 -0
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/METADATA +3 -1
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/RECORD +13 -12
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/WHEEL +1 -1
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/entry_points.txt +0 -0
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/licenses/LICENSE +0 -0
- {anemoi_utils-0.4.23.dist-info → anemoi_utils-0.4.25.dist-info}/top_level.txt +0 -0
anemoi/utils/_version.py
CHANGED
anemoi/utils/cli.py
CHANGED
|
@@ -15,6 +15,7 @@ import os
|
|
|
15
15
|
import sys
|
|
16
16
|
import traceback
|
|
17
17
|
from typing import Callable
|
|
18
|
+
from typing import Optional
|
|
18
19
|
|
|
19
20
|
try:
|
|
20
21
|
import argcomplete
|
|
@@ -185,7 +186,9 @@ def register_commands(here: str, package: str, select: Callable, fail: Callable
|
|
|
185
186
|
return result
|
|
186
187
|
|
|
187
188
|
|
|
188
|
-
def cli_main(
|
|
189
|
+
def cli_main(
|
|
190
|
+
version: str, description: str, commands: dict[str, Command], test_arguments: Optional[list[str]] = None
|
|
191
|
+
) -> None:
|
|
189
192
|
"""Main entry point for the CLI.
|
|
190
193
|
|
|
191
194
|
Parameters
|
|
@@ -196,9 +199,11 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
|
|
|
196
199
|
The description of the CLI
|
|
197
200
|
commands : dict[str, Command]
|
|
198
201
|
A dictionary of command names to Command instances
|
|
202
|
+
test_arguments : list[str], optional
|
|
203
|
+
The command line arguments to parse, used for testing purposes, by default None
|
|
199
204
|
"""
|
|
200
205
|
parser = make_parser(description, commands)
|
|
201
|
-
args, unknown = parser.parse_known_args()
|
|
206
|
+
args, unknown = parser.parse_known_args(test_arguments)
|
|
202
207
|
if argcomplete:
|
|
203
208
|
argcomplete.autocomplete(parser)
|
|
204
209
|
|
|
@@ -220,7 +225,7 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
|
|
|
220
225
|
|
|
221
226
|
if unknown and not cmd.accept_unknown_args:
|
|
222
227
|
# This should trigger an error
|
|
223
|
-
parser.parse_args()
|
|
228
|
+
parser.parse_args(test_arguments)
|
|
224
229
|
|
|
225
230
|
try:
|
|
226
231
|
if unknown:
|
|
@@ -228,9 +233,14 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
|
|
|
228
233
|
else:
|
|
229
234
|
cmd.run(args)
|
|
230
235
|
except ValueError as e:
|
|
236
|
+
|
|
237
|
+
if test_arguments:
|
|
238
|
+
raise
|
|
239
|
+
|
|
231
240
|
traceback.print_exc()
|
|
232
241
|
LOG.error("\n💣 %s", str(e).lstrip())
|
|
233
242
|
LOG.error("💣 Exiting")
|
|
234
243
|
sys.exit(1)
|
|
235
244
|
|
|
236
|
-
|
|
245
|
+
if not test_arguments:
|
|
246
|
+
sys.exit(0)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# (C) Copyright 2024 Anemoi contributors.
|
|
2
|
+
#
|
|
3
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
4
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
5
|
+
#
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
from argparse import ArgumentParser
|
|
12
|
+
from argparse import Namespace
|
|
13
|
+
|
|
14
|
+
from anemoi.utils.remote import transfer
|
|
15
|
+
|
|
16
|
+
from . import Command
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Transfer(Command):
|
|
20
|
+
"""Handle configuration related commands."""
|
|
21
|
+
|
|
22
|
+
def add_arguments(self, command_parser: ArgumentParser) -> None:
|
|
23
|
+
"""Add arguments to the command parser.
|
|
24
|
+
|
|
25
|
+
Parameters
|
|
26
|
+
----------
|
|
27
|
+
command_parser : ArgumentParser
|
|
28
|
+
The argument parser to which the arguments will be added.
|
|
29
|
+
"""
|
|
30
|
+
command_parser.add_argument(
|
|
31
|
+
"--source", help="A path to a local file or folder or a URL to a file or a folder on S3."
|
|
32
|
+
)
|
|
33
|
+
command_parser.add_argument(
|
|
34
|
+
"--target", help="A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder."
|
|
35
|
+
)
|
|
36
|
+
command_parser.add_argument(
|
|
37
|
+
"--overwrite",
|
|
38
|
+
action="store_true",
|
|
39
|
+
help="If the data is already on in the target location it will be overwritten..",
|
|
40
|
+
)
|
|
41
|
+
command_parser.add_argument(
|
|
42
|
+
"--resume",
|
|
43
|
+
action="store_true",
|
|
44
|
+
help="If the data is already on S3 it will not be uploaded, unless the remote file has a different size.",
|
|
45
|
+
)
|
|
46
|
+
command_parser.add_argument("--verbosity", default=1, help="The level of verbosity, by default 1.")
|
|
47
|
+
command_parser.add_argument(
|
|
48
|
+
"--progress", default=None, help="A callable that will be called with the number of files."
|
|
49
|
+
)
|
|
50
|
+
command_parser.add_argument(
|
|
51
|
+
"--threads", default=1, help="The number of threads to use when uploading a directory, by default 1."
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def run(self, args: Namespace) -> None:
|
|
55
|
+
"""Execute the command with the provided arguments.
|
|
56
|
+
|
|
57
|
+
Parameters
|
|
58
|
+
----------
|
|
59
|
+
args : Namespace
|
|
60
|
+
The arguments passed to the command.
|
|
61
|
+
"""
|
|
62
|
+
transfer(
|
|
63
|
+
source=args.source,
|
|
64
|
+
target=args.target,
|
|
65
|
+
overwrite=args.overwrite,
|
|
66
|
+
resume=args.resume,
|
|
67
|
+
verbosity=args.verbosity,
|
|
68
|
+
progress=args.progress,
|
|
69
|
+
threads=args.threads,
|
|
70
|
+
temporary_target=False,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
command = Transfer
|
anemoi/utils/grids.py
CHANGED
|
@@ -17,9 +17,12 @@ from typing import List
|
|
|
17
17
|
from typing import Tuple
|
|
18
18
|
from typing import Union
|
|
19
19
|
|
|
20
|
+
import deprecation
|
|
20
21
|
import numpy as np
|
|
21
22
|
import requests
|
|
22
23
|
|
|
24
|
+
from anemoi.utils._version import __version__
|
|
25
|
+
|
|
23
26
|
from .caching import cached
|
|
24
27
|
|
|
25
28
|
LOG = logging.getLogger(__name__)
|
|
@@ -28,6 +31,12 @@ LOG = logging.getLogger(__name__)
|
|
|
28
31
|
GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
|
|
29
32
|
|
|
30
33
|
|
|
34
|
+
@deprecation.deprecated(
|
|
35
|
+
deprecated_in="0.4.25",
|
|
36
|
+
removed_in="0.5.0",
|
|
37
|
+
current_version=__version__,
|
|
38
|
+
details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
|
|
39
|
+
)
|
|
31
40
|
def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
32
41
|
"""Convert Cartesian coordinates to latitude and longitude.
|
|
33
42
|
|
|
@@ -51,6 +60,12 @@ def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarr
|
|
|
51
60
|
)
|
|
52
61
|
|
|
53
62
|
|
|
63
|
+
@deprecation.deprecated(
|
|
64
|
+
deprecated_in="0.4.25",
|
|
65
|
+
removed_in="0.5.0",
|
|
66
|
+
current_version=__version__,
|
|
67
|
+
details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
|
|
68
|
+
)
|
|
54
69
|
def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
55
70
|
"""Convert latitude and longitude to Cartesian coordinates.
|
|
56
71
|
|
|
@@ -87,6 +102,12 @@ def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tupl
|
|
|
87
102
|
return x, y, z
|
|
88
103
|
|
|
89
104
|
|
|
105
|
+
@deprecation.deprecated(
|
|
106
|
+
deprecated_in="0.4.25",
|
|
107
|
+
removed_in="0.5.0",
|
|
108
|
+
current_version=__version__,
|
|
109
|
+
details="Use anemoi.transform.spatial.nearest_grid_points instead.",
|
|
110
|
+
)
|
|
90
111
|
def nearest_grid_points(
|
|
91
112
|
source_latitudes: np.ndarray,
|
|
92
113
|
source_longitudes: np.ndarray,
|
|
@@ -169,6 +190,12 @@ def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
|
|
|
169
190
|
return response.content
|
|
170
191
|
|
|
171
192
|
|
|
193
|
+
@deprecation.deprecated(
|
|
194
|
+
deprecated_in="0.4.25",
|
|
195
|
+
removed_in="0.5.0",
|
|
196
|
+
current_version=__version__,
|
|
197
|
+
details="Use anemoi.transform.grids.named.lookup instead.",
|
|
198
|
+
)
|
|
172
199
|
def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
|
|
173
200
|
"""Load grid data by name.
|
|
174
201
|
|
anemoi/utils/remote/__init__.py
CHANGED
|
@@ -371,7 +371,12 @@ class BaseDownload(Loader):
|
|
|
371
371
|
The target path.
|
|
372
372
|
"""
|
|
373
373
|
if os.path.exists(target):
|
|
374
|
-
|
|
374
|
+
if os.path.isfile(target):
|
|
375
|
+
LOGGER.info(f"Deleting file {target}")
|
|
376
|
+
os.remove(target)
|
|
377
|
+
else:
|
|
378
|
+
LOGGER.info(f"Deleting directory {target}")
|
|
379
|
+
shutil.rmtree(target)
|
|
375
380
|
|
|
376
381
|
|
|
377
382
|
class BaseUpload(Loader):
|
|
@@ -642,10 +647,10 @@ def transfer(
|
|
|
642
647
|
A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder.
|
|
643
648
|
The url should start with 's3://' or 'ssh://'.
|
|
644
649
|
overwrite : bool, optional
|
|
645
|
-
If the data is
|
|
650
|
+
If the data is already on in the target location it will be overwritten.
|
|
646
651
|
By default False
|
|
647
652
|
resume : bool, optional
|
|
648
|
-
If the data is
|
|
653
|
+
If the data is already on S3 it will not be uploaded, unless the remote file has a different size
|
|
649
654
|
Ignored if the target is an SSH remote folder (ssh://).
|
|
650
655
|
By default False
|
|
651
656
|
verbosity : int, optional
|
anemoi/utils/remote/s3.py
CHANGED
|
@@ -24,6 +24,7 @@ the `~/.config/anemoi/settings.toml`
|
|
|
24
24
|
or `~/.config/anemoi/settings-secrets.toml` files.
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
|
+
import fnmatch
|
|
27
28
|
import logging
|
|
28
29
|
import os
|
|
29
30
|
import threading
|
|
@@ -38,15 +39,15 @@ from ..humanize import bytes_to_human
|
|
|
38
39
|
from . import BaseDownload
|
|
39
40
|
from . import BaseUpload
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
LOG = logging.getLogger(__name__)
|
|
43
|
+
SECRETS = ["aws_access_key_id", "aws_secret_access_key"]
|
|
43
44
|
|
|
44
45
|
# s3_clients are not thread-safe, so we need to create a new client for each thread
|
|
45
46
|
|
|
46
47
|
thread_local = threading.local()
|
|
47
48
|
|
|
48
49
|
|
|
49
|
-
def s3_client(bucket: str, region: str = None) -> Any:
|
|
50
|
+
def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
|
|
50
51
|
"""Get an S3 client for the specified bucket and region.
|
|
51
52
|
|
|
52
53
|
Parameters
|
|
@@ -55,6 +56,8 @@ def s3_client(bucket: str, region: str = None) -> Any:
|
|
|
55
56
|
The name of the S3 bucket.
|
|
56
57
|
region : str, optional
|
|
57
58
|
The AWS region of the S3 bucket.
|
|
59
|
+
service : str, optional
|
|
60
|
+
The AWS service to use, default is "s3".
|
|
58
61
|
|
|
59
62
|
Returns
|
|
60
63
|
-------
|
|
@@ -68,7 +71,7 @@ def s3_client(bucket: str, region: str = None) -> Any:
|
|
|
68
71
|
if not hasattr(thread_local, "s3_clients"):
|
|
69
72
|
thread_local.s3_clients = {}
|
|
70
73
|
|
|
71
|
-
key = f"{bucket}-{region}"
|
|
74
|
+
key = f"{bucket}-{region}-{service}"
|
|
72
75
|
|
|
73
76
|
if key in thread_local.s3_clients:
|
|
74
77
|
return thread_local.s3_clients[key]
|
|
@@ -96,17 +99,27 @@ def s3_client(bucket: str, region: str = None) -> Any:
|
|
|
96
99
|
# We may be accessing a different S3 compatible service
|
|
97
100
|
# Use anemoi.config to get the configuration
|
|
98
101
|
|
|
99
|
-
|
|
100
|
-
|
|
102
|
+
region = "unknown-region"
|
|
103
|
+
|
|
104
|
+
options = {"region_name": region}
|
|
105
|
+
config = load_config(secrets=SECRETS)
|
|
101
106
|
|
|
102
107
|
cfg = config.get("object-storage", {})
|
|
108
|
+
candidate = None
|
|
103
109
|
for k, v in cfg.items():
|
|
104
110
|
if isinstance(v, (str, int, float, bool)):
|
|
105
111
|
options[k] = v
|
|
106
112
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
113
|
+
if isinstance(v, dict):
|
|
114
|
+
if fnmatch.fnmatch(bucket, k):
|
|
115
|
+
if candidate is not None:
|
|
116
|
+
raise ValueError(f"Multiple object storage configurations match {bucket}: {candidate} and {k}")
|
|
117
|
+
candidate = k
|
|
118
|
+
|
|
119
|
+
if candidate is not None:
|
|
120
|
+
for k, v in cfg.get(candidate, {}).items():
|
|
121
|
+
if isinstance(v, (str, int, float, bool)):
|
|
122
|
+
options[k] = v
|
|
110
123
|
|
|
111
124
|
type = options.pop("type", "s3")
|
|
112
125
|
if type != "s3":
|
|
@@ -115,11 +128,27 @@ def s3_client(bucket: str, region: str = None) -> Any:
|
|
|
115
128
|
if "config" in options:
|
|
116
129
|
boto3_config.update(options["config"])
|
|
117
130
|
del options["config"]
|
|
118
|
-
from botocore.client import Config
|
|
119
131
|
|
|
120
132
|
options["config"] = Config(**boto3_config)
|
|
121
133
|
|
|
122
|
-
|
|
134
|
+
def _(options):
|
|
135
|
+
|
|
136
|
+
def __(k, v):
|
|
137
|
+
if k in SECRETS:
|
|
138
|
+
return "***"
|
|
139
|
+
return v
|
|
140
|
+
|
|
141
|
+
if isinstance(options, dict):
|
|
142
|
+
return {k: __(k, v) for k, v in options.items()}
|
|
143
|
+
|
|
144
|
+
if isinstance(options, list):
|
|
145
|
+
return [_(o) for o in options]
|
|
146
|
+
|
|
147
|
+
return options
|
|
148
|
+
|
|
149
|
+
LOG.debug(f"Using S3 options: {_(options)}")
|
|
150
|
+
|
|
151
|
+
thread_local.s3_clients[key] = boto3.client(service, **options)
|
|
123
152
|
|
|
124
153
|
return thread_local.s3_clients[key]
|
|
125
154
|
|
|
@@ -215,7 +244,7 @@ class S3Upload(BaseUpload):
|
|
|
215
244
|
size = os.path.getsize(source)
|
|
216
245
|
|
|
217
246
|
if verbosity > 0:
|
|
218
|
-
|
|
247
|
+
LOG.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
|
|
219
248
|
|
|
220
249
|
try:
|
|
221
250
|
results = s3.head_object(Bucket=bucket, Key=key)
|
|
@@ -227,7 +256,7 @@ class S3Upload(BaseUpload):
|
|
|
227
256
|
|
|
228
257
|
if remote_size is not None:
|
|
229
258
|
if remote_size != size:
|
|
230
|
-
|
|
259
|
+
LOG.warning(
|
|
231
260
|
f"{target} already exists, but with different size, re-uploading (remote={remote_size}, local={size})"
|
|
232
261
|
)
|
|
233
262
|
elif resume:
|
|
@@ -400,7 +429,7 @@ class S3Download(BaseDownload):
|
|
|
400
429
|
size = int(response["ContentLength"])
|
|
401
430
|
|
|
402
431
|
if verbosity > 0:
|
|
403
|
-
|
|
432
|
+
LOG.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
|
|
404
433
|
|
|
405
434
|
if overwrite:
|
|
406
435
|
resume = False
|
|
@@ -409,7 +438,7 @@ class S3Download(BaseDownload):
|
|
|
409
438
|
if os.path.exists(target):
|
|
410
439
|
local_size = os.path.getsize(target)
|
|
411
440
|
if local_size != size:
|
|
412
|
-
|
|
441
|
+
LOG.warning(
|
|
413
442
|
f"{target} already with different size, re-downloading (remote={size}, local={local_size})"
|
|
414
443
|
)
|
|
415
444
|
else:
|
|
@@ -464,7 +493,7 @@ def _list_objects(target: str, batch: bool = False) -> Iterable:
|
|
|
464
493
|
yield from objects
|
|
465
494
|
|
|
466
495
|
|
|
467
|
-
def
|
|
496
|
+
def delete_folder(target: str) -> None:
|
|
468
497
|
"""Delete a folder from S3.
|
|
469
498
|
|
|
470
499
|
Parameters
|
|
@@ -477,13 +506,13 @@ def _delete_folder(target: str) -> None:
|
|
|
477
506
|
|
|
478
507
|
total = 0
|
|
479
508
|
for batch in _list_objects(target, batch=True):
|
|
480
|
-
|
|
509
|
+
LOG.info(f"Deleting {len(batch):,} objects from {target}")
|
|
481
510
|
s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": o["Key"]} for o in batch]})
|
|
482
511
|
total += len(batch)
|
|
483
|
-
|
|
512
|
+
LOG.info(f"Deleted {len(batch):,} objects (total={total:,})")
|
|
484
513
|
|
|
485
514
|
|
|
486
|
-
def
|
|
515
|
+
def delete_file(target: str) -> None:
|
|
487
516
|
"""Delete a file from S3.
|
|
488
517
|
|
|
489
518
|
Parameters
|
|
@@ -505,12 +534,12 @@ def _delete_file(target: str) -> None:
|
|
|
505
534
|
exits = False
|
|
506
535
|
|
|
507
536
|
if not exits:
|
|
508
|
-
|
|
537
|
+
LOG.warning(f"{target} does not exist. Did you mean to delete a folder? Then add a trailing '/'")
|
|
509
538
|
return
|
|
510
539
|
|
|
511
|
-
|
|
540
|
+
LOG.info(f"Deleting {target}")
|
|
512
541
|
s3.delete_object(Bucket=bucket, Key=key)
|
|
513
|
-
|
|
542
|
+
LOG.info(f"{target} is deleted")
|
|
514
543
|
|
|
515
544
|
|
|
516
545
|
def delete(target: str) -> None:
|
|
@@ -525,9 +554,9 @@ def delete(target: str) -> None:
|
|
|
525
554
|
assert target.startswith("s3://")
|
|
526
555
|
|
|
527
556
|
if target.endswith("/"):
|
|
528
|
-
|
|
557
|
+
delete_folder(target)
|
|
529
558
|
else:
|
|
530
|
-
|
|
559
|
+
delete_file(target)
|
|
531
560
|
|
|
532
561
|
|
|
533
562
|
def list_folder(folder: str) -> Iterable:
|
|
@@ -581,7 +610,33 @@ def object_info(target: str) -> dict:
|
|
|
581
610
|
return s3.head_object(Bucket=bucket, Key=key)
|
|
582
611
|
except s3.exceptions.ClientError as e:
|
|
583
612
|
if e.response["Error"]["Code"] == "404":
|
|
584
|
-
raise
|
|
613
|
+
raise FileNotFoundError(f"{target} does not exist")
|
|
614
|
+
raise
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def object_exists(target: str) -> bool:
|
|
618
|
+
"""Check if an object exists.
|
|
619
|
+
|
|
620
|
+
Parameters
|
|
621
|
+
----------
|
|
622
|
+
target : str
|
|
623
|
+
The URL of a file or a folder on S3. The URL should start with 's3://'.
|
|
624
|
+
|
|
625
|
+
Returns
|
|
626
|
+
-------
|
|
627
|
+
bool
|
|
628
|
+
True if the object exists, False otherwise.
|
|
629
|
+
"""
|
|
630
|
+
|
|
631
|
+
_, _, bucket, key = target.split("/", 3)
|
|
632
|
+
s3 = s3_client(bucket)
|
|
633
|
+
|
|
634
|
+
try:
|
|
635
|
+
s3.head_object(Bucket=bucket, Key=key)
|
|
636
|
+
return True
|
|
637
|
+
except s3.exceptions.ClientError as e:
|
|
638
|
+
if e.response["Error"]["Code"] == "404":
|
|
639
|
+
return False
|
|
585
640
|
raise
|
|
586
641
|
|
|
587
642
|
|
|
@@ -600,7 +655,7 @@ def object_acl(target: str) -> dict:
|
|
|
600
655
|
"""
|
|
601
656
|
|
|
602
657
|
_, _, bucket, key = target.split("/", 3)
|
|
603
|
-
s3 = s3_client()
|
|
658
|
+
s3 = s3_client(bucket)
|
|
604
659
|
|
|
605
660
|
return s3.get_object_acl(Bucket=bucket, Key=key)
|
|
606
661
|
|
|
@@ -643,3 +698,29 @@ def upload(source: str, target: str, *args, **kwargs) -> None:
|
|
|
643
698
|
|
|
644
699
|
assert target.startswith("s3://"), f"target {target} should start with 's3://'"
|
|
645
700
|
return transfer(source, target, *args, **kwargs)
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def quotas(target: str) -> dict:
|
|
704
|
+
"""Get the quotas for an S3 bucket.
|
|
705
|
+
|
|
706
|
+
Parameters
|
|
707
|
+
----------
|
|
708
|
+
target : str
|
|
709
|
+
The URL of a file or a folder on S3. The URL should start with 's3://'.
|
|
710
|
+
|
|
711
|
+
Returns
|
|
712
|
+
-------
|
|
713
|
+
dict
|
|
714
|
+
A dictionary with the quotas for the bucket.
|
|
715
|
+
"""
|
|
716
|
+
from botocore.exceptions import ClientError
|
|
717
|
+
|
|
718
|
+
_, _, bucket, _ = target.split("/", 3)
|
|
719
|
+
s3 = s3_client(bucket, service="service-quotas")
|
|
720
|
+
|
|
721
|
+
try:
|
|
722
|
+
return s3.list_service_quotas(ServiceCode="ec2")
|
|
723
|
+
except ClientError as e:
|
|
724
|
+
if e.response["Error"]["Code"] == "404":
|
|
725
|
+
raise ValueError(f"{target} does not exist")
|
|
726
|
+
raise
|
anemoi/utils/testing.py
CHANGED
|
@@ -299,3 +299,104 @@ def skip_missing_packages(*names: str) -> callable:
|
|
|
299
299
|
return pytest.mark.skipif(True, reason=f"Package {missing[0]} is not installed")
|
|
300
300
|
|
|
301
301
|
return pytest.mark.skipif(True, reason=f"Packages {list_to_human(missing)} are not installed")
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def skip_if_missing_command(cmd: str) -> callable:
|
|
305
|
+
"""Skip a test if the specified command is not available.
|
|
306
|
+
|
|
307
|
+
Parameters
|
|
308
|
+
----------
|
|
309
|
+
cmd : str
|
|
310
|
+
The name of the command to check.
|
|
311
|
+
|
|
312
|
+
Returns
|
|
313
|
+
-------
|
|
314
|
+
Callable
|
|
315
|
+
A decorator that skips the test if the specified command is not available.
|
|
316
|
+
"""
|
|
317
|
+
|
|
318
|
+
import shutil
|
|
319
|
+
|
|
320
|
+
if shutil.which(cmd):
|
|
321
|
+
return lambda f: f
|
|
322
|
+
|
|
323
|
+
return pytest.mark.skipif(True, reason=f"Command '{cmd}' is not available")
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def cli_testing(package: str, cmd: str, *args: list[str]) -> None:
|
|
327
|
+
"""Run a CLI command for testing purposes.
|
|
328
|
+
|
|
329
|
+
Parameters
|
|
330
|
+
----------
|
|
331
|
+
package : str
|
|
332
|
+
The name of the package containing the CLI commands.
|
|
333
|
+
Can be 'anemoi-datasets' or 'anemoi.datasets'.
|
|
334
|
+
cmd : str
|
|
335
|
+
The command to run.
|
|
336
|
+
*args : list[str]
|
|
337
|
+
Additional arguments to pass to the command.
|
|
338
|
+
"""
|
|
339
|
+
|
|
340
|
+
package = package.replace("-", ".")
|
|
341
|
+
COMMANDS = getattr(__import__(f"{package}.commands", fromlist=["COMMANDS"]), "COMMANDS")
|
|
342
|
+
version = getattr(__import__(f"{package}._version", fromlist=["__version__"]), "__version__", "0.1.0")
|
|
343
|
+
|
|
344
|
+
from anemoi.utils.cli import cli_main
|
|
345
|
+
|
|
346
|
+
cli_main(
|
|
347
|
+
version=version,
|
|
348
|
+
description=f"Testing the '{cmd}' CLI command from the '{package}' package.",
|
|
349
|
+
commands=COMMANDS,
|
|
350
|
+
test_arguments=(cmd,) + args,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def run_tests(globals: dict[str, callable]) -> None:
|
|
355
|
+
"""Run all test functions that start with 'test_'.
|
|
356
|
+
|
|
357
|
+
Parameters
|
|
358
|
+
----------
|
|
359
|
+
globals : dict[str, callable]
|
|
360
|
+
The global namespace containing the test functions.
|
|
361
|
+
|
|
362
|
+
Example
|
|
363
|
+
-------
|
|
364
|
+
|
|
365
|
+
Call from a test file to run all tests in that file:
|
|
366
|
+
|
|
367
|
+
```python
|
|
368
|
+
if __name__ == "__main__":
|
|
369
|
+
from anemoi.utils.testing import run_tests
|
|
370
|
+
run_tests(globals())
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Useful for debugging or running tests in an interactive environment.
|
|
374
|
+
|
|
375
|
+
"""
|
|
376
|
+
import logging
|
|
377
|
+
|
|
378
|
+
import rich
|
|
379
|
+
|
|
380
|
+
logging.basicConfig(level=logging.INFO)
|
|
381
|
+
|
|
382
|
+
for name, obj in list(globals.items()):
|
|
383
|
+
if name.startswith("test_") and callable(obj):
|
|
384
|
+
|
|
385
|
+
pytestmark = getattr(obj, "pytestmark", None)
|
|
386
|
+
if pytestmark is not None:
|
|
387
|
+
if not isinstance(pytestmark, list):
|
|
388
|
+
pytestmark = [pytestmark]
|
|
389
|
+
|
|
390
|
+
skip = False
|
|
391
|
+
for m in pytestmark:
|
|
392
|
+
if m.name == "skipif" and m.args == (True,):
|
|
393
|
+
skip = True
|
|
394
|
+
rich.print(
|
|
395
|
+
f"[red]Skipping [bold]{name}[/bold] due to skipif condition [bold]{m.kwargs['reason']}[/bold].[/red]"
|
|
396
|
+
)
|
|
397
|
+
break
|
|
398
|
+
if skip:
|
|
399
|
+
continue
|
|
400
|
+
|
|
401
|
+
rich.print(f"[green]Running [bold]{name}[/bold]...[/green]")
|
|
402
|
+
obj()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: anemoi-utils
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.25
|
|
4
4
|
Summary: A package to hold various functions to support training of ML models on ECMWF data.
|
|
5
5
|
Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
|
|
6
6
|
License: Apache License
|
|
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
|
225
225
|
Requires-Python: >=3.9
|
|
226
226
|
License-File: LICENSE
|
|
227
227
|
Requires-Dist: aniso8601
|
|
228
|
+
Requires-Dist: deprecation
|
|
228
229
|
Requires-Dist: importlib-metadata; python_version < "3.10"
|
|
229
230
|
Requires-Dist: multiurl
|
|
230
231
|
Requires-Dist: numpy
|
|
231
232
|
Requires-Dist: pydantic>=2.9
|
|
232
233
|
Requires-Dist: python-dateutil
|
|
233
234
|
Requires-Dist: pyyaml
|
|
235
|
+
Requires-Dist: rich
|
|
234
236
|
Requires-Dist: tomli; python_version < "3.11"
|
|
235
237
|
Requires-Dist: tqdm
|
|
236
238
|
Provides-Extra: all
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
|
|
2
2
|
anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
|
|
3
|
-
anemoi/utils/_version.py,sha256=
|
|
3
|
+
anemoi/utils/_version.py,sha256=3hQuMRH6SNq4Gy0jS57VxD61lFAwagLT8wP_tfZa6fA,513
|
|
4
4
|
anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
|
|
5
5
|
anemoi/utils/checkpoints.py,sha256=N4WpAZXa4etrpSEKhHqUUtG2-x9w3FJMHcLO-dDAXPY,9600
|
|
6
|
-
anemoi/utils/cli.py,sha256=
|
|
6
|
+
anemoi/utils/cli.py,sha256=Cd3ESQkNWecbGnJjkR_SKHdFPETJWFrHqg5ovtANDKs,6522
|
|
7
7
|
anemoi/utils/compatibility.py,sha256=wRBRMmxQP88rNcWiP5gqXliwYQbBv1iCAsDjcCRi5UY,2234
|
|
8
8
|
anemoi/utils/config.py,sha256=EEfcSxW2CD6fFOzDtqz_uYlMKuYq4X5QinJW_8GBYj4,17325
|
|
9
9
|
anemoi/utils/dates.py,sha256=CnY6JOdpk0T-waPEowMRTkcMzxcN0GcjPVtLkwH_byw,17196
|
|
10
10
|
anemoi/utils/devtools.py,sha256=W3OBu96MkXRIl7Qh1SE5Zd6aB1R0QlnmlrlpBYM0fVY,3527
|
|
11
11
|
anemoi/utils/grib.py,sha256=201WcxjjAl92Y2HX2kZ2S8Qr5dN-oG7nV-vQLaybzP4,3610
|
|
12
|
-
anemoi/utils/grids.py,sha256=
|
|
12
|
+
anemoi/utils/grids.py,sha256=VlhuN8MZDBu-G50QFI1XHV9IXLway2LpvI9X8sDjb5s,6047
|
|
13
13
|
anemoi/utils/hindcasts.py,sha256=iYVIxSNFL2HJcc_k1abCFLkpJFGHT8WKRIR4wcAwA3s,2144
|
|
14
14
|
anemoi/utils/humanize.py,sha256=pjnFJAKHbEAOfcvn8c48kt-8eFy6FGW_U2ruJvfamrA,25189
|
|
15
15
|
anemoi/utils/logs.py,sha256=naTgrmPwWHD4eekFttXftS4gtcAGYHpCqG4iwYprNDA,1804
|
|
@@ -19,23 +19,24 @@ anemoi/utils/rules.py,sha256=VspUoPmw7tijrs6l_wl4vDjr_zVQsFjx9ITiBSvxgc8,6972
|
|
|
19
19
|
anemoi/utils/s3.py,sha256=xMT48kbcelcjjqsaU567WI3oZ5eqo88Rlgyx5ECszAU,4074
|
|
20
20
|
anemoi/utils/sanitise.py,sha256=ZYGdSX6qihQANr3pHZjbKnoapnzP1KcrWdW1Ul1mOGk,3668
|
|
21
21
|
anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
|
|
22
|
-
anemoi/utils/testing.py,sha256=
|
|
22
|
+
anemoi/utils/testing.py,sha256=1qisyMA71HMT2mQUpLOsh01F9KR6vIOLid5WVyniaiw,10605
|
|
23
23
|
anemoi/utils/text.py,sha256=HkzIvi24obDceFLpJEwBJ9PmPrJUkQN2TrElJ-A87gU,14441
|
|
24
24
|
anemoi/utils/timer.py,sha256=_leKMYza2faM7JKlGE7LCNy13rbdPnwaCF7PSrI_NmI,3895
|
|
25
25
|
anemoi/utils/commands/__init__.py,sha256=5u_6EwdqYczIAgJfCwRSyQAYFEqh2ZuHHT57g9g7sdI,808
|
|
26
26
|
anemoi/utils/commands/config.py,sha256=nYvYbjcum9uumRa3gzPfmQCjNZKURXMA2XOUwz9b7ls,1369
|
|
27
27
|
anemoi/utils/commands/requests.py,sha256=AEbssF1OlpbmSwrV5Lj6amCCn0w_-nbajBWTwYV15vA,2059
|
|
28
|
+
anemoi/utils/commands/transfer.py,sha256=05GT5BAY9xt6vx86--xfJ8bkPCUkIRz0d265hDcJLCE,2600
|
|
28
29
|
anemoi/utils/mars/__init__.py,sha256=b-Lc3L1TAQd9ODs0Z1YSJzgZCO1K_M3DSgx_yd2qXvM,2724
|
|
29
30
|
anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
|
|
30
31
|
anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew,1142
|
|
31
|
-
anemoi/utils/remote/__init__.py,sha256=
|
|
32
|
-
anemoi/utils/remote/s3.py,sha256=
|
|
32
|
+
anemoi/utils/remote/__init__.py,sha256=7nHu-LRspYW2Fx9GNLjsxpytAUIvhIbOjb0Xmxb-33s,20988
|
|
33
|
+
anemoi/utils/remote/s3.py,sha256=VgxNuJpJBnfxwkkV9TyrSkf7fjIuHqy1YG9vRdj9QRc,19978
|
|
33
34
|
anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
|
|
34
35
|
anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
|
|
35
36
|
anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
|
|
36
|
-
anemoi_utils-0.4.
|
|
37
|
-
anemoi_utils-0.4.
|
|
38
|
-
anemoi_utils-0.4.
|
|
39
|
-
anemoi_utils-0.4.
|
|
40
|
-
anemoi_utils-0.4.
|
|
41
|
-
anemoi_utils-0.4.
|
|
37
|
+
anemoi_utils-0.4.25.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
|
|
38
|
+
anemoi_utils-0.4.25.dist-info/METADATA,sha256=PILyf6_K2AZs161wasDnRvfhMdQjm4Cub1YzXkdKFmM,15486
|
|
39
|
+
anemoi_utils-0.4.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
40
|
+
anemoi_utils-0.4.25.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
|
|
41
|
+
anemoi_utils-0.4.25.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
|
|
42
|
+
anemoi_utils-0.4.25.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|