ddsapi 0.7.1__tar.gz → 2026.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddsapi-2026.6.1/PKG-INFO +74 -0
- ddsapi-2026.6.1/README.md +41 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi/__init__.py +2 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi/api.py +105 -23
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi/cache.py +5 -3
- ddsapi-2026.6.1/ddsapi/version.py +1 -0
- ddsapi-2026.6.1/ddsapi.egg-info/PKG-INFO +74 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi.egg-info/SOURCES.txt +7 -3
- ddsapi-2026.6.1/ddsapi.egg-info/requires.txt +15 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi.egg-info/top_level.txt +0 -1
- ddsapi-2026.6.1/pyproject.toml +49 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/tests/test_config.py +10 -2
- ddsapi-2026.6.1/tests/test_errors.py +56 -0
- ddsapi-2026.6.1/tests/test_integration.py +25 -0
- ddsapi-2026.6.1/tests/test_optional.py +29 -0
- ddsapi-2026.6.1/tests/test_version.py +9 -0
- ddsapi-0.7.1/PKG-INFO +0 -56
- ddsapi-0.7.1/README.md +0 -24
- ddsapi-0.7.1/ddsapi.egg-info/PKG-INFO +0 -56
- ddsapi-0.7.1/ddsapi.egg-info/requires.txt +0 -1
- ddsapi-0.7.1/setup.py +0 -52
- ddsapi-0.7.1/tests/__init__.py +0 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/LICENSE +0 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/ddsapi.egg-info/dependency_links.txt +0 -0
- {ddsapi-0.7.1 → ddsapi-2026.6.1}/setup.cfg +0 -0
ddsapi-2026.6.1/PKG-INFO
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ddsapi
|
|
3
|
+
Version: 2026.6.1
|
|
4
|
+
Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
|
|
5
|
+
Author-email: CMCC Data Delivery System Team <dds-support@cmcc.it>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/CMCC-Foundation/ddsapi-client/
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Environment :: Web Environment
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Natural Language :: English
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: requests
|
|
20
|
+
Requires-Dist: urllib3
|
|
21
|
+
Provides-Extra: data
|
|
22
|
+
Requires-Dist: xarray; extra == "data"
|
|
23
|
+
Requires-Dist: dask; extra == "data"
|
|
24
|
+
Requires-Dist: zarr; extra == "data"
|
|
25
|
+
Requires-Dist: netCDF4; extra == "data"
|
|
26
|
+
Requires-Dist: h5netcdf; extra == "data"
|
|
27
|
+
Requires-Dist: cf-xarray; extra == "data"
|
|
28
|
+
Requires-Dist: requests; extra == "data"
|
|
29
|
+
Requires-Dist: aiohttp; extra == "data"
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: pytest; extra == "test"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# DDSAPI-Client
|
|
35
|
+
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
36
|
+
|
|
37
|
+
## Requirements
|
|
38
|
+
Python 3.11 or greater is required
|
|
39
|
+
|
|
40
|
+
### Installation
|
|
41
|
+
|
|
42
|
+
The base client is lightweight (HTTP only). The libraries needed to **open** the
|
|
43
|
+
retrieved datasets (xarray, zarr, netCDF4, …) are optional — install them only if you
|
|
44
|
+
need to work with the downloaded data.
|
|
45
|
+
|
|
46
|
+
#### conda
|
|
47
|
+
|
|
48
|
+
From the `cmcc-dds` channel:
|
|
49
|
+
```bash
|
|
50
|
+
$ conda install -c cmcc-dds ddsapi
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
To also open the downloaded datasets, add the data stack (from conda-forge):
|
|
54
|
+
```bash
|
|
55
|
+
$ conda install -c conda-forge xarray zarr netcdf4 h5netcdf dask cf-xarray
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
#### pip
|
|
59
|
+
```bash
|
|
60
|
+
$ pip install ddsapi
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To also open the downloaded datasets, install the optional `data` extra:
|
|
64
|
+
```bash
|
|
65
|
+
$ pip install 'ddsapi[data]'
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Configuration
|
|
69
|
+
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
url: https://ddshub.cmcc.it/api/v2
|
|
73
|
+
key: <api-key>
|
|
74
|
+
```
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# DDSAPI-Client
|
|
2
|
+
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
3
|
+
|
|
4
|
+
## Requirements
|
|
5
|
+
Python 3.11 or greater is required
|
|
6
|
+
|
|
7
|
+
### Installation
|
|
8
|
+
|
|
9
|
+
The base client is lightweight (HTTP only). The libraries needed to **open** the
|
|
10
|
+
retrieved datasets (xarray, zarr, netCDF4, …) are optional — install them only if you
|
|
11
|
+
need to work with the downloaded data.
|
|
12
|
+
|
|
13
|
+
#### conda
|
|
14
|
+
|
|
15
|
+
From the `cmcc-dds` channel:
|
|
16
|
+
```bash
|
|
17
|
+
$ conda install -c cmcc-dds ddsapi
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
To also open the downloaded datasets, add the data stack (from conda-forge):
|
|
21
|
+
```bash
|
|
22
|
+
$ conda install -c conda-forge xarray zarr netcdf4 h5netcdf dask cf-xarray
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
#### pip
|
|
26
|
+
```bash
|
|
27
|
+
$ pip install ddsapi
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
To also open the downloaded datasets, install the optional `data` extra:
|
|
31
|
+
```bash
|
|
32
|
+
$ pip install 'ddsapi[data]'
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Configuration
|
|
36
|
+
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
url: https://ddshub.cmcc.it/api/v2
|
|
40
|
+
key: <api-key>
|
|
41
|
+
```
|
|
@@ -29,13 +29,57 @@ import zipfile
|
|
|
29
29
|
import shutil
|
|
30
30
|
from typing import Any, Union, Iterable
|
|
31
31
|
import urllib3
|
|
32
|
-
import xarray as xr
|
|
33
|
-
from geokube import open_datacube
|
|
34
32
|
|
|
35
33
|
from .cache import CacheManager
|
|
36
34
|
|
|
37
35
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
38
36
|
|
|
37
|
+
|
|
38
|
+
class DDSAPIError(Exception):
|
|
39
|
+
"""A clean, user-facing error raised by the CMCC DDS API client.
|
|
40
|
+
|
|
41
|
+
Carries a concise message (HTTP status, reason and server detail) and is
|
|
42
|
+
raised with the underlying low-level traceback suppressed (``from None``),
|
|
43
|
+
so notebooks show only the relevant error instead of a long internal stack.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _require_xarray():
|
|
48
|
+
"""Import xarray on demand, with a clear hint if the optional deps are missing.
|
|
49
|
+
|
|
50
|
+
The data libraries (xarray and its engines) are optional: the base client can
|
|
51
|
+
submit requests and report errors without them. They are only needed to open
|
|
52
|
+
the resulting datasets.
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
import xarray as xr
|
|
56
|
+
except ImportError:
|
|
57
|
+
raise DDSAPIError(
|
|
58
|
+
"This operation needs the optional data dependencies (xarray plus the "
|
|
59
|
+
"relevant engine, e.g. zarr or netCDF4). Install them with "
|
|
60
|
+
"`pip install 'ddsapi[data]'` or "
|
|
61
|
+
"`conda install -c conda-forge xarray zarr netcdf4`."
|
|
62
|
+
) from None
|
|
63
|
+
return xr
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _zarr_v2_open_kwarg(xr):
|
|
67
|
+
"""Kwarg that pins the zarr reader to format v2.
|
|
68
|
+
|
|
69
|
+
DDS writes Zarr v2 stores; with zarr-python 3 installed the engine would
|
|
70
|
+
otherwise assume v3. xarray renamed ``zarr_version`` -> ``zarr_format``, so
|
|
71
|
+
pick whichever the installed backend supports.
|
|
72
|
+
"""
|
|
73
|
+
import inspect
|
|
74
|
+
from xarray.backends.zarr import ZarrBackendEntrypoint
|
|
75
|
+
params = inspect.signature(ZarrBackendEntrypoint.open_dataset).parameters
|
|
76
|
+
if "zarr_format" in params:
|
|
77
|
+
return {"zarr_format": 2}
|
|
78
|
+
if "zarr_version" in params:
|
|
79
|
+
return {"zarr_version": 2}
|
|
80
|
+
return {}
|
|
81
|
+
|
|
82
|
+
|
|
39
83
|
def str2bool(s: str, default: Any = False) -> bool:
|
|
40
84
|
if isinstance(s, str):
|
|
41
85
|
return s.lower() in ("1", "yes", "y", "true")
|
|
@@ -176,7 +220,8 @@ class _Result:
|
|
|
176
220
|
return [os.path.join(self.target_path, f) for f in os.listdir(self.target_path)]
|
|
177
221
|
return self.target_path
|
|
178
222
|
|
|
179
|
-
def dataset(self):
|
|
223
|
+
def dataset(self, engine):
|
|
224
|
+
xr = _require_xarray()
|
|
180
225
|
self.debug("Location %s", self.target_path)
|
|
181
226
|
if self.target_path is None:
|
|
182
227
|
raise RuntimeError(
|
|
@@ -187,12 +232,14 @@ class _Result:
|
|
|
187
232
|
if os.path.isdir(self.target_path):
|
|
188
233
|
ds_list = []
|
|
189
234
|
for f in os.listdir(self.target_path):
|
|
190
|
-
ds =
|
|
235
|
+
#ds = open_datacube(os.path.join(self.target_path, f), engine=engine, decode_coords='all', chunks="auto")
|
|
236
|
+
ds = xr.open_dataset(os.path.join(self.target_path, f), engine=engine, decode_coords='all', chunks="auto")
|
|
191
237
|
ds_list.append(ds)
|
|
192
238
|
if len(ds_list) == 1:
|
|
193
239
|
return ds_list[0]
|
|
194
240
|
return ds_list
|
|
195
|
-
return
|
|
241
|
+
#return open_datacube(self.target_path, engine=engine, decode_coords='all', chunks="auto")
|
|
242
|
+
return xr.open_dataset(self.target_path, engine=engine, decode_coords='all', chunks="auto")
|
|
196
243
|
|
|
197
244
|
class EnvVarNames:
|
|
198
245
|
RC_FILE: str = "DDSAPI_RC"
|
|
@@ -405,6 +452,7 @@ class Client:
|
|
|
405
452
|
delete=self.delete,
|
|
406
453
|
),
|
|
407
454
|
)
|
|
455
|
+
self.format = 'netcdf'
|
|
408
456
|
|
|
409
457
|
def _submit(self, url, request):
|
|
410
458
|
session = self.session
|
|
@@ -426,24 +474,38 @@ class Client:
|
|
|
426
474
|
reply = self.robust(session.get)(url, verify=self.verify)
|
|
427
475
|
return reply
|
|
428
476
|
|
|
477
|
+
@staticmethod
|
|
478
|
+
def _response_body(response):
|
|
479
|
+
"""Best-effort body: parsed JSON if possible, else stripped raw text."""
|
|
480
|
+
try:
|
|
481
|
+
return response.json()
|
|
482
|
+
except requests.exceptions.JSONDecodeError:
|
|
483
|
+
return (response.text or "").strip()
|
|
484
|
+
|
|
429
485
|
def _maybe_handle_fails_on_4xx_code(self, response):
|
|
430
|
-
if response
|
|
431
|
-
|
|
432
|
-
|
|
486
|
+
if response is None:
|
|
487
|
+
raise DDSAPIError(
|
|
488
|
+
"DDS server returned no response (connection failed after retries)."
|
|
489
|
+
) from None
|
|
490
|
+
if response.status_code >= 300:
|
|
491
|
+
body = self._response_body(response)
|
|
492
|
+
detail = body.get("detail") if isinstance(body, dict) else body
|
|
493
|
+
raise DDSAPIError(
|
|
494
|
+
f"DDS request failed: HTTP {response.status_code} {response.reason} "
|
|
495
|
+
f"at {response.url}\n{detail}"
|
|
496
|
+
) from None
|
|
433
497
|
return False
|
|
434
498
|
|
|
435
499
|
def _maybe_handle_fails_on_200_code(self, response):
|
|
436
|
-
if response.status_code == 200:
|
|
437
|
-
msg =
|
|
500
|
+
if response is not None and response.status_code == 200:
|
|
501
|
+
msg = self._response_body(response)
|
|
438
502
|
if isinstance(msg, int):
|
|
439
503
|
self.logger.info("Request is scheduled with ID: %s", msg)
|
|
440
504
|
return False
|
|
441
|
-
if msg
|
|
442
|
-
|
|
443
|
-
"
|
|
444
|
-
|
|
445
|
-
)
|
|
446
|
-
return True
|
|
505
|
+
if isinstance(msg, dict) and msg.get("status") == "FAILED":
|
|
506
|
+
raise DDSAPIError(
|
|
507
|
+
f"DDS request failed: {msg.get('fail_reason')}"
|
|
508
|
+
) from None
|
|
447
509
|
return False
|
|
448
510
|
|
|
449
511
|
def retrieve(self, dataset_id, product_id, request, target=None):
|
|
@@ -651,6 +713,7 @@ class Client:
|
|
|
651
713
|
return cached_target
|
|
652
714
|
if request["format"] == "zarr":
|
|
653
715
|
streaming = True
|
|
716
|
+
self.format = request["format"]
|
|
654
717
|
jreply = self._submit(
|
|
655
718
|
f"{self.url}/datasets/{dataset_id}/{product_id}/execute",
|
|
656
719
|
request,
|
|
@@ -683,7 +746,7 @@ class Client:
|
|
|
683
746
|
)
|
|
684
747
|
if self._maybe_handle_fails_on_200_code(reply):
|
|
685
748
|
return
|
|
686
|
-
msg =
|
|
749
|
+
msg = self._response_body(reply)
|
|
687
750
|
self.debug("REPLY %s", reply)
|
|
688
751
|
|
|
689
752
|
if reply.status_code != 200:
|
|
@@ -720,16 +783,35 @@ class Client:
|
|
|
720
783
|
target=result.get_files(), overwrite=False)
|
|
721
784
|
else:
|
|
722
785
|
self.info(f"Opening zarr in streaming.... {result._download_url}")
|
|
723
|
-
return
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
786
|
+
#return open_datacube(result._download_url, decode_coords='all', engine="zarr", chunks='auto')
|
|
787
|
+
xr = _require_xarray()
|
|
788
|
+
return xr.open_dataset(
|
|
789
|
+
result._download_url,
|
|
790
|
+
decode_coords='all',
|
|
791
|
+
engine="zarr",
|
|
792
|
+
chunks={},
|
|
793
|
+
storage_options={
|
|
794
|
+
"client_kwargs": {"headers": self.auth_token}
|
|
795
|
+
},
|
|
796
|
+
**_zarr_v2_open_kwarg(xr),
|
|
797
|
+
)
|
|
798
|
+
except DDSAPIError:
|
|
799
|
+
raise
|
|
800
|
+
except Exception as err:
|
|
801
|
+
raise DDSAPIError(
|
|
802
|
+
f"Failed to download/open the result for request "
|
|
803
|
+
f"{request_id}: {err}"
|
|
804
|
+
) from None
|
|
805
|
+
|
|
728
806
|
if target is not None:
|
|
729
807
|
return result
|
|
730
808
|
else:
|
|
731
809
|
if self.cache.disabled:
|
|
732
|
-
|
|
810
|
+
if self.format == "netcdf":
|
|
811
|
+
engine = "netcdf4"
|
|
812
|
+
else:
|
|
813
|
+
engine = self.format
|
|
814
|
+
return result.dataset(engine)
|
|
733
815
|
else:
|
|
734
816
|
return self.cache.maybe_get_from_cache(dataset_id=dataset_id, product_id=product_id, request=request)
|
|
735
817
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""This module contains simple query-hash-based cache mechanism."""
|
|
2
|
+
from __future__ import annotations
|
|
2
3
|
|
|
3
4
|
import os
|
|
4
5
|
import json
|
|
@@ -8,7 +9,6 @@ from dataclasses import dataclass, field
|
|
|
8
9
|
import pickle
|
|
9
10
|
import shutil
|
|
10
11
|
|
|
11
|
-
import xarray as xr
|
|
12
12
|
import hashlib
|
|
13
13
|
|
|
14
14
|
|
|
@@ -93,15 +93,17 @@ class CacheManager:
|
|
|
93
93
|
return
|
|
94
94
|
if not target:
|
|
95
95
|
return
|
|
96
|
+
from .api import _require_xarray
|
|
97
|
+
xr = _require_xarray()
|
|
96
98
|
if isinstance(target, list):
|
|
97
99
|
ds_list = []
|
|
98
100
|
for f in target:
|
|
99
|
-
ds = xr.open_dataset(os.path.join(self.cache_dir, f), decode_coords='all')
|
|
101
|
+
ds = xr.open_dataset(os.path.join(self.cache_dir, f), decode_coords='all', chunks='auto')
|
|
100
102
|
ds_list.append(ds)
|
|
101
103
|
if len(ds_list) == 1:
|
|
102
104
|
return ds_list[0]
|
|
103
105
|
return ds_list
|
|
104
106
|
elif isinstance(target, str):
|
|
105
|
-
return xr.open_dataset(os.path.join(self.cache_dir, target), decode_coords='all')
|
|
107
|
+
return xr.open_dataset(os.path.join(self.cache_dir, target), decode_coords='all', chunks='auto')
|
|
106
108
|
else:
|
|
107
109
|
raise TypeError
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2026.06.1"
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ddsapi
|
|
3
|
+
Version: 2026.6.1
|
|
4
|
+
Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
|
|
5
|
+
Author-email: CMCC Data Delivery System Team <dds-support@cmcc.it>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/CMCC-Foundation/ddsapi-client/
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Environment :: Web Environment
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Natural Language :: English
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: requests
|
|
20
|
+
Requires-Dist: urllib3
|
|
21
|
+
Provides-Extra: data
|
|
22
|
+
Requires-Dist: xarray; extra == "data"
|
|
23
|
+
Requires-Dist: dask; extra == "data"
|
|
24
|
+
Requires-Dist: zarr; extra == "data"
|
|
25
|
+
Requires-Dist: netCDF4; extra == "data"
|
|
26
|
+
Requires-Dist: h5netcdf; extra == "data"
|
|
27
|
+
Requires-Dist: cf-xarray; extra == "data"
|
|
28
|
+
Requires-Dist: requests; extra == "data"
|
|
29
|
+
Requires-Dist: aiohttp; extra == "data"
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: pytest; extra == "test"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# DDSAPI-Client
|
|
35
|
+
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
36
|
+
|
|
37
|
+
## Requirements
|
|
38
|
+
Python 3.11 or greater is required
|
|
39
|
+
|
|
40
|
+
### Installation
|
|
41
|
+
|
|
42
|
+
The base client is lightweight (HTTP only). The libraries needed to **open** the
|
|
43
|
+
retrieved datasets (xarray, zarr, netCDF4, …) are optional — install them only if you
|
|
44
|
+
need to work with the downloaded data.
|
|
45
|
+
|
|
46
|
+
#### conda
|
|
47
|
+
|
|
48
|
+
From the `cmcc-dds` channel:
|
|
49
|
+
```bash
|
|
50
|
+
$ conda install -c cmcc-dds ddsapi
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
To also open the downloaded datasets, add the data stack (from conda-forge):
|
|
54
|
+
```bash
|
|
55
|
+
$ conda install -c conda-forge xarray zarr netcdf4 h5netcdf dask cf-xarray
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
#### pip
|
|
59
|
+
```bash
|
|
60
|
+
$ pip install ddsapi
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To also open the downloaded datasets, install the optional `data` extra:
|
|
64
|
+
```bash
|
|
65
|
+
$ pip install 'ddsapi[data]'
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Configuration
|
|
69
|
+
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
url: https://ddshub.cmcc.it/api/v2
|
|
73
|
+
key: <api-key>
|
|
74
|
+
```
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
README.md
|
|
3
|
-
|
|
3
|
+
pyproject.toml
|
|
4
4
|
ddsapi/__init__.py
|
|
5
5
|
ddsapi/api.py
|
|
6
6
|
ddsapi/cache.py
|
|
7
|
+
ddsapi/version.py
|
|
7
8
|
ddsapi.egg-info/PKG-INFO
|
|
8
9
|
ddsapi.egg-info/SOURCES.txt
|
|
9
10
|
ddsapi.egg-info/dependency_links.txt
|
|
10
11
|
ddsapi.egg-info/requires.txt
|
|
11
12
|
ddsapi.egg-info/top_level.txt
|
|
12
|
-
tests/
|
|
13
|
-
tests/
|
|
13
|
+
tests/test_config.py
|
|
14
|
+
tests/test_errors.py
|
|
15
|
+
tests/test_integration.py
|
|
16
|
+
tests/test_optional.py
|
|
17
|
+
tests/test_version.py
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ddsapi"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Python Client to access and download data from CMCC Data Delivery System (DDS)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{name = "CMCC Data Delivery System Team", email = "dds-support@cmcc.it"},
|
|
15
|
+
]
|
|
16
|
+
dependencies = [
|
|
17
|
+
"requests",
|
|
18
|
+
"urllib3",
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Development Status :: 4 - Beta",
|
|
22
|
+
"Environment :: Web Environment",
|
|
23
|
+
"Intended Audience :: Science/Research",
|
|
24
|
+
"Natural Language :: English",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Hydrology",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
# Heavy data libraries needed only to open the retrieved datasets.
|
|
33
|
+
data = ["xarray", "dask", "zarr", "netCDF4", "h5netcdf", "cf-xarray", "requests", "aiohttp"]
|
|
34
|
+
test = ["pytest"]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/CMCC-Foundation/ddsapi-client/"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.dynamic]
|
|
40
|
+
version = {attr = "ddsapi.version.__version__"}
|
|
41
|
+
|
|
42
|
+
[tool.setuptools.packages.find]
|
|
43
|
+
include = ["ddsapi*"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
markers = [
|
|
48
|
+
"integration: requires a live DDS server (set DDSAPI_URL and DDSAPI_KEY)",
|
|
49
|
+
]
|
|
@@ -58,7 +58,12 @@ def test_get_cache_from_env_var(read_config):
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
@mock.patch("ddsapi.api.read_config", _read_config_mock)
|
|
61
|
-
def test_read_conf_from_rc_file():
|
|
61
|
+
def test_read_conf_from_rc_file(tmp_path, monkeypatch):
|
|
62
|
+
# read_config is mocked, but Config only reads the rc file if it exists, so
|
|
63
|
+
# point DDSAPI_RC at a real (empty) temp file to make the test hermetic.
|
|
64
|
+
rc = tmp_path / ".ddsapirc"
|
|
65
|
+
rc.write_text("")
|
|
66
|
+
monkeypatch.setenv(EnvVarNames.RC_FILE, str(rc))
|
|
62
67
|
conf = Config()
|
|
63
68
|
assert conf.key == "key_from_file"
|
|
64
69
|
assert conf.url == "url_from_file"
|
|
@@ -68,7 +73,10 @@ def test_read_conf_from_rc_file():
|
|
|
68
73
|
|
|
69
74
|
@mock.patch.dict(os.environ, {EnvVarNames.CACHEDIR: "dds-cache"})
|
|
70
75
|
@mock.patch("ddsapi.api.read_config", _read_config_mock2)
|
|
71
|
-
def test_conf_from_file_overwrites():
|
|
76
|
+
def test_conf_from_file_overwrites(tmp_path, monkeypatch):
|
|
77
|
+
rc = tmp_path / ".ddsapirc"
|
|
78
|
+
rc.write_text("")
|
|
79
|
+
monkeypatch.setenv(EnvVarNames.RC_FILE, str(rc))
|
|
72
80
|
conf = Config()
|
|
73
81
|
assert conf.key == "key_from_file"
|
|
74
82
|
assert conf.url == "url_from_file"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from ddsapi import DDSAPIError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_response_body_returns_json(client, fake_response):
|
|
7
|
+
assert client._response_body(fake_response(200, body={"a": 1})) == {"a": 1}
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_response_body_text_fallback(client, fake_response):
|
|
11
|
+
r = fake_response(500, text="<html>oops</html>", raise_json=True)
|
|
12
|
+
assert client._response_body(r) == "<html>oops</html>"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.mark.parametrize("status", [400, 403, 404, 500])
|
|
16
|
+
def test_4xx_raises_clean_ddsapierror(client, fake_response, status):
|
|
17
|
+
r = fake_response(status, reason="Err", body={"detail": "boom"})
|
|
18
|
+
with pytest.raises(DDSAPIError) as ei:
|
|
19
|
+
client._maybe_handle_fails_on_4xx_code(r)
|
|
20
|
+
assert f"HTTP {status}" in str(ei.value)
|
|
21
|
+
assert "boom" in str(ei.value)
|
|
22
|
+
# raised with `from None` -> no chained traceback shown to the user
|
|
23
|
+
assert ei.value.__suppress_context__ is True
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_4xx_none_response_raises(client):
|
|
27
|
+
with pytest.raises(DDSAPIError):
|
|
28
|
+
client._maybe_handle_fails_on_4xx_code(None)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize("status", [200, 201, 204])
|
|
32
|
+
def test_4xx_ok_returns_false(client, fake_response, status):
|
|
33
|
+
assert client._maybe_handle_fails_on_4xx_code(fake_response(status)) is False
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_200_failed_job_raises(client, fake_response):
|
|
37
|
+
r = fake_response(200, body={"status": "FAILED", "fail_reason": "bad variables"})
|
|
38
|
+
with pytest.raises(DDSAPIError) as ei:
|
|
39
|
+
client._maybe_handle_fails_on_200_code(r)
|
|
40
|
+
assert "bad variables" in str(ei.value)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_200_scheduled_id_returns_false(client, fake_response):
|
|
44
|
+
assert client._maybe_handle_fails_on_200_code(fake_response(200, body=42)) is False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_retrieve_surfaces_clean_error(client, fake_response, monkeypatch):
|
|
48
|
+
r = fake_response(
|
|
49
|
+
400, reason="Bad Request",
|
|
50
|
+
body={"detail": "format 'geojson' is not allowed for this product"},
|
|
51
|
+
)
|
|
52
|
+
monkeypatch.setattr(client, "_submit", lambda *a, **k: r)
|
|
53
|
+
with pytest.raises(DDSAPIError) as ei:
|
|
54
|
+
client.retrieve("era5-downscaled", "hourly", {"format": "netcdf"})
|
|
55
|
+
assert "HTTP 400" in str(ei.value)
|
|
56
|
+
assert "geojson" in str(ei.value)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
pytestmark = pytest.mark.integration
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.skipif(
|
|
9
|
+
not (os.environ.get("DDSAPI_URL") and os.environ.get("DDSAPI_KEY")),
|
|
10
|
+
reason="requires a live DDS server (set DDSAPI_URL and DDSAPI_KEY)",
|
|
11
|
+
)
|
|
12
|
+
def test_retrieve_small_request():
|
|
13
|
+
import ddsapi
|
|
14
|
+
|
|
15
|
+
xr = pytest.importorskip("xarray")
|
|
16
|
+
c = ddsapi.Client() # url/key read from DDSAPI_URL / DDSAPI_KEY
|
|
17
|
+
ds = c.retrieve(
|
|
18
|
+
"era5-downscaled",
|
|
19
|
+
"hourly",
|
|
20
|
+
{
|
|
21
|
+
"time": {"start": "2020-01-01T00:00:00", "stop": "2020-01-01T00:59:59"},
|
|
22
|
+
"format": "zarr",
|
|
23
|
+
},
|
|
24
|
+
)
|
|
25
|
+
assert isinstance(ds, xr.Dataset)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import ddsapi
|
|
6
|
+
from ddsapi import DDSAPIError
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_import_and_client_do_not_require_xarray():
|
|
10
|
+
# The base client must import and construct without the optional data stack.
|
|
11
|
+
assert hasattr(ddsapi, "Client")
|
|
12
|
+
assert hasattr(ddsapi, "DDSAPIError")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_require_xarray_missing_raises_clean_error(monkeypatch):
|
|
16
|
+
# Simulate xarray not being importable.
|
|
17
|
+
monkeypatch.setitem(sys.modules, "xarray", None)
|
|
18
|
+
with pytest.raises(DDSAPIError) as ei:
|
|
19
|
+
ddsapi.api._require_xarray()
|
|
20
|
+
assert "ddsapi[data]" in str(ei.value)
|
|
21
|
+
assert ei.value.__suppress_context__ is True
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_zarr_v2_open_kwarg_pins_v2():
|
|
25
|
+
# DDS stores are Zarr v2; the open kwarg must pin the reader to format 2.
|
|
26
|
+
xr = pytest.importorskip("xarray")
|
|
27
|
+
kw = ddsapi.api._zarr_v2_open_kwarg(xr)
|
|
28
|
+
assert kw in ({"zarr_format": 2}, {"zarr_version": 2})
|
|
29
|
+
assert list(kw.values()) == [2]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import ddsapi
|
|
2
|
+
import ddsapi.version
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_version_is_single_sourced():
|
|
6
|
+
# __version__ comes from ddsapi/version.py (the single source of truth).
|
|
7
|
+
assert ddsapi.__version__ == ddsapi.version.__version__
|
|
8
|
+
assert isinstance(ddsapi.__version__, str)
|
|
9
|
+
assert ddsapi.__version__
|
ddsapi-0.7.1/PKG-INFO
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: ddsapi
|
|
3
|
-
Version: 0.7.1
|
|
4
|
-
Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
|
|
5
|
-
Home-page: https://github.com/CMCC-Foundation/ddsapi-client/
|
|
6
|
-
Author: CMCC Data Delivery System Team
|
|
7
|
-
Author-email: dds-support@cmcc.it
|
|
8
|
-
License: Apache License, Version 2.0
|
|
9
|
-
Classifier: Development Status :: 4 - Beta
|
|
10
|
-
Classifier: Environment :: Web Environment
|
|
11
|
-
Classifier: Intended Audience :: Science/Research
|
|
12
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
-
Classifier: Natural Language :: English
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
17
|
-
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
18
|
-
Requires-Python: >=3.11
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
License-File: LICENSE
|
|
21
|
-
Requires-Dist: geokube==0.2.7.1
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: home-page
|
|
28
|
-
Dynamic: license
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
|
-
|
|
33
|
-
# DDSAPI-Client
|
|
34
|
-
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
35
|
-
|
|
36
|
-
## Requirements
|
|
37
|
-
Python 3.7, Python 3.8
|
|
38
|
-
|
|
39
|
-
### Installation
|
|
40
|
-
Conda Installation
|
|
41
|
-
```bash
|
|
42
|
-
$ conda install -c fondazione-cmcc ddsapi
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
Pip installation
|
|
46
|
-
```bash
|
|
47
|
-
$ pip install ddsapi
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
### Configuration
|
|
51
|
-
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
52
|
-
|
|
53
|
-
```bash
|
|
54
|
-
url: https://ddshub.cmcc.it/api/v2
|
|
55
|
-
key: <api-key>
|
|
56
|
-
```
|
ddsapi-0.7.1/README.md
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
# DDSAPI-Client
|
|
2
|
-
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
3
|
-
|
|
4
|
-
## Requirements
|
|
5
|
-
Python 3.7, Python 3.8
|
|
6
|
-
|
|
7
|
-
### Installation
|
|
8
|
-
Conda Installation
|
|
9
|
-
```bash
|
|
10
|
-
$ conda install -c fondazione-cmcc ddsapi
|
|
11
|
-
```
|
|
12
|
-
|
|
13
|
-
Pip installation
|
|
14
|
-
```bash
|
|
15
|
-
$ pip install ddsapi
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
### Configuration
|
|
19
|
-
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
20
|
-
|
|
21
|
-
```bash
|
|
22
|
-
url: https://ddshub.cmcc.it/api/v2
|
|
23
|
-
key: <api-key>
|
|
24
|
-
```
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: ddsapi
|
|
3
|
-
Version: 0.7.1
|
|
4
|
-
Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
|
|
5
|
-
Home-page: https://github.com/CMCC-Foundation/ddsapi-client/
|
|
6
|
-
Author: CMCC Data Delivery System Team
|
|
7
|
-
Author-email: dds-support@cmcc.it
|
|
8
|
-
License: Apache License, Version 2.0
|
|
9
|
-
Classifier: Development Status :: 4 - Beta
|
|
10
|
-
Classifier: Environment :: Web Environment
|
|
11
|
-
Classifier: Intended Audience :: Science/Research
|
|
12
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
-
Classifier: Natural Language :: English
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
17
|
-
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
18
|
-
Requires-Python: >=3.11
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
License-File: LICENSE
|
|
21
|
-
Requires-Dist: geokube==0.2.7.1
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: home-page
|
|
28
|
-
Dynamic: license
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
|
-
|
|
33
|
-
# DDSAPI-Client
|
|
34
|
-
Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
|
|
35
|
-
|
|
36
|
-
## Requirements
|
|
37
|
-
Python 3.7, Python 3.8
|
|
38
|
-
|
|
39
|
-
### Installation
|
|
40
|
-
Conda Installation
|
|
41
|
-
```bash
|
|
42
|
-
$ conda install -c fondazione-cmcc ddsapi
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
Pip installation
|
|
46
|
-
```bash
|
|
47
|
-
$ pip install ddsapi
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
### Configuration
|
|
51
|
-
To use the tool a file `$HOME/.ddsapirc` must be created as following
|
|
52
|
-
|
|
53
|
-
```bash
|
|
54
|
-
url: https://ddshub.cmcc.it/api/v2
|
|
55
|
-
key: <api-key>
|
|
56
|
-
```
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
geokube==0.2.7.1
|
ddsapi-0.7.1/setup.py
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
"""Packaging utility for CMCC DDSAPI"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
# https://packaging.python.org/tutorials/packaging-projects/
|
|
5
|
-
# https://packaging.python.org/discussions/install-requires-vs-requirements/
|
|
6
|
-
# https://pypi.org/classifiers/
|
|
7
|
-
# https://setuptools.readthedocs.io/en/latest/setuptools.html
|
|
8
|
-
|
|
9
|
-
# Run this file with the command:
|
|
10
|
-
# $ python setup.py install
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
import setuptools
|
|
14
|
-
|
|
15
|
-
with open("README.md", "r") as f:
|
|
16
|
-
long_description = f.read()
|
|
17
|
-
|
|
18
|
-
setuptools.setup(
|
|
19
|
-
name="ddsapi",
|
|
20
|
-
version="0.7.1",
|
|
21
|
-
author="CMCC Data Delivery System Team",
|
|
22
|
-
author_email="dds-support@cmcc.it",
|
|
23
|
-
description=(
|
|
24
|
-
"Python Client to access and download data from CMCC Data Delivery"
|
|
25
|
-
" System (DDS)"
|
|
26
|
-
),
|
|
27
|
-
long_description=long_description,
|
|
28
|
-
long_description_content_type="text/markdown",
|
|
29
|
-
url="https://github.com/CMCC-Foundation/ddsapi-client/",
|
|
30
|
-
packages=setuptools.find_packages(),
|
|
31
|
-
install_requires=[
|
|
32
|
-
"geokube==0.2.7.1"
|
|
33
|
-
#"netCDF4>=1.5.3",
|
|
34
|
-
#"scipy>=1.5.2",
|
|
35
|
-
#"requests>=2.23.0",
|
|
36
|
-
#"xarray==2022.10.0",
|
|
37
|
-
#"numpy==1.26.0"
|
|
38
|
-
],
|
|
39
|
-
classifiers=[
|
|
40
|
-
"Development Status :: 4 - Beta",
|
|
41
|
-
"Environment :: Web Environment",
|
|
42
|
-
"Intended Audience :: Science/Research",
|
|
43
|
-
"License :: OSI Approved :: Apache Software License",
|
|
44
|
-
"Natural Language :: English",
|
|
45
|
-
"Operating System :: OS Independent",
|
|
46
|
-
"Programming Language :: Python :: 3",
|
|
47
|
-
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
|
48
|
-
"Topic :: Scientific/Engineering :: Hydrology",
|
|
49
|
-
],
|
|
50
|
-
python_requires=">=3.11",
|
|
51
|
-
license="Apache License, Version 2.0",
|
|
52
|
-
)
|
ddsapi-0.7.1/tests/__init__.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|