ddsapi 0.6b0__tar.gz → 0.6b3__tar.gz

{ddsapi-0.6b0 → ddsapi-0.6b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddsapi
-Version: 0.6b0
+Version: 0.6b3
 Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
 Home-page: https://github.com/CMCC-Foundation/ddsapi-client/
 Author: CMCC Data Delivery System Team
@@ -18,6 +18,10 @@ Classifier: Topic :: Scientific/Engineering :: Hydrology
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: netCDF4>=1.5.3
+Requires-Dist: scipy>=1.5.2
+Requires-Dist: requests>=2.23.0
+Requires-Dist: xarray>=0.16.0
 
 # DDSAPI-Client
 Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
@@ -40,6 +44,6 @@ $ pip install ddsapi
 To use the tool a file `$HOME/.ddsapirc` must be created as following
 
 ```bash
-url: https://ddsapi.cmcc.it/v1
+url: https://ddshub.cmcc.it/api/v2
 key: <api-key>
 ```

{ddsapi-0.6b0 → ddsapi-0.6b3}/README.md

@@ -19,6 +19,6 @@ $ pip install ddsapi
 To use the tool a file `$HOME/.ddsapirc` must be created as following
 
 ```bash
-url: https://ddsapi.cmcc.it/v1
+url: https://ddshub.cmcc.it/api/v2
 key: <api-key>
 ```

{ddsapi-0.6b0 → ddsapi-0.6b3}/ddsapi/api.py

@@ -15,17 +15,30 @@ from __future__ import (
     division,
     print_function,
     unicode_literals,
+    annotations
 )
+import dis
 
+import pickle
 import json
 import time
 import os
 import logging
 import requests
 import zipfile
-
+import shutil
+from typing import Any, Union, Iterable
+import urllib3
 import xarray as xr
 
+from .cache import CacheManager
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+def str2bool(s: str, default: Any = False) -> bool:
+    if isinstance(s, str):
+        return s.lower() in ("1", "yes", "y", "true")
+    return default
 
 def bytes_to_string(n):
     u = ["", "K", "M", "G", "T", "P"]
@@ -149,6 +162,18 @@ class _Result:
     def download(self, target=None):
         self.debug("Downloading from %s", self._download_url)
         return self._download(self._download_url, target)
+    
+    def _maybe_unzip(self):
+        if self._is_zip:
+            with zipfile.ZipFile(self.target_path, "r") as zip_ref:
+                self.target_path = self.target_path.split(".")[0]
+                zip_ref.extractall(self.target_path)
+
+    def get_files(self) -> Union[str, Iterable[str]]:
+        self._maybe_unzip()
+        if os.path.isdir(self.target_path):
+            return [os.path.join(self.target_path, f) for f in os.listdir(self.target_path)]
+        return self.target_path
 
     def dataset(self):
         self.debug("Location %s", self.target_path)
@@ -157,10 +182,7 @@ class _Result:
                 "No file found. Downloading could have not finished yet or it"
                 " failed."
             )
-        if self._is_zip:
-            with zipfile.ZipFile(self.target_path, "r") as zip_ref:
-                self.target_path = self.target_path.split(".")[0]
-                zip_ref.extractall(self.target_path)
+        self._maybe_unzip()
         if os.path.isdir(self.target_path):
             ds_list = []
             for f in os.listdir(self.target_path):
@@ -171,6 +193,70 @@ class _Result:
             return ds_list
 
         return xr.open_dataset(self.target_path)
+    
+class EnvVarNames:
+    RC_FILE: str = "DDSAPI_RC"
+    URL: str = "DDSAPI_URL"
+    KEY: str = "DDSAPI_KEY"
+    CACHEDIR: str = "DDSAPI_CLIENT_CACHE_DIR"  
+    DISABLE_CACHE: str = "DDS_CACHE_DISABLE"  
+
+class Config:
+    """Configuration class.
+
+    The configuration defined in the `.ddsapirc` file overwrites those
+    defined using environmental variables."""
+
+    _RC_FILENAME: str = ".ddsapirc"
+    _CACHE_FILENAME: str = ".cache"
+    url: None | str = None
+    key: None | str = None
+    cachedir: None | str = None
+    verify: None | int = None
+
+    def __init__(self, url: str | None = None, key: str | None = None) -> None:
+        dotrc = os.environ.get(
+            EnvVarNames.RC_FILE,
+            os.path.expanduser(os.path.join("~", Config._RC_FILENAME)),
+        )
+        self._init_conf_with_env_vars()
+        self._maybe_overwrite_from_rc_file(dotrc)
+        self._maybe_apply_defaults()
+        self._apply_user_defined(url=url, key=key)
+
+    def _apply_user_defined(self, url, key) -> None:
+        if url:
+            self.url = url
+        if key:
+            self.key = key
+
+    def _maybe_apply_defaults(self) -> None:
+        if self.verify is None:
+            self.verify = 1
+        if not self.cachedir:
+            self.cachedir = os.path.expanduser(
+                os.path.join("~", Config._CACHE_FILENAME)
+            )
+
+    def _init_conf_with_env_vars(self) -> None:
+        self.url = os.environ.get(EnvVarNames.URL)
+        self.key = os.environ.get(EnvVarNames.KEY)
+        self.cachedir = os.environ.get(EnvVarNames.CACHEDIR)
+
+    def _maybe_overwrite_from_rc_file(self, rcfile: str) -> None:
+        if os.path.exists(rcfile): 
+            config = read_config(rcfile)
+        else:
+            config = {}
+        if not self.key:
+            self.key = config.get("key")
+        if not self.url:
+            self.url = config.get("url")
+        if not self.verify:
+            verify_ = config.get("verify")
+            self.verify = int(verify_) if verify_ else None
+        if not self.cachedir:
+            self.cachedir = config.get("cachedir")
 
 
 class Client:
@@ -241,8 +327,8 @@ class Client:
 
     def __init__(
         self,
-        url=os.environ.get("DDSAPI_URL"),
-        key=os.environ.get("DDSAPI_KEY"),
+        url=None,
+        key=None,
         direct_path=False,
         quiet=False,
         debug=False,
@@ -267,27 +353,18 @@ class Client:
             logging.basicConfig(
                 level=level, format="%(asctime)s %(levelname)s %(message)s"
             )
+        config = Config()
+        self.url = config.url
+        self.key = config.key
+        self.verify = config.verify
+        self.cache: CacheManager = CacheManager(
+            cache_dir=config.cachedir,
+            disabled=str2bool(os.environ.get(EnvVarNames.DISABLE_CACHE), default=True))
+        self.cachedir = config.cachedir
+
+        if self.url is None or self.key is None:
+            raise Exception("Missing/incomplete configuration file")
 
-        dotrc = os.environ.get("DDSAPI_RC", os.path.expanduser("~/.ddsapirc"))
-
-        if url is None or key is None:
-            if os.path.exists(dotrc):
-                config = read_config(dotrc)
-
-                if key is None:
-                    key = config.get("key")
-
-                if url is None:
-                    url = config.get("url")
-
-                if verify is None:
-                    verify = int(config.get("verify", 1))
-
-        if url is None or key is None or key is None:
-            raise Exception(f"Missing/incomplete configuration file: {dotrc}")
-
-        self.url = url
-        self.key = key
         self.direct_path = direct_path
 
         self.quiet = quiet
@@ -562,6 +639,10 @@ class Client:
         session = self.session
         if "format" not in request:
             request["temp_file"] = target
+        cached_target = self.cache.maybe_get_from_cache(dataset_id=dataset_id, product_id=product_id, request=request)
+        if cached_target:
+            return cached_target
+
         jreply = self._submit(
             f"{self.url}/datasets/{dataset_id}/{product_id}/execute",
             request,
@@ -616,14 +697,29 @@ class Client:
                 result = _Result(
                     self, request_id=request_id, auth_token=self.auth_token
                 )
+                import tempfile
+                if target is None:
+                    _target = tempfile.NamedTemporaryFile(delete=False).name
+                else:
+                    _target = target
+                try:
+                    result.download(_target)
+                    self.cache.add_to_cache(
+                        dataset_id=dataset_id, 
+                        product_id=product_id, 
+                        request=request, 
+                        target=result.get_files(), overwrite=False)
+                except RuntimeError as err:
+                    self.logger.error(str(err))
+                    return
+    
                 if target is not None:
-                    try:
-                        result.download(target)
-                    except RuntimeError as err:
-                        self.logger.error(str(err))
-                        return
                     return result
-                return result.dataset()
+                else:
+                    if self.cache.disabled:
+                        return result.dataset()
+                    else:
+                        return self.cache.maybe_get_from_cache(dataset_id=dataset_id, product_id=product_id, request=request)
 
             if msg["status"] == "RUNNING" or msg["status"] == "PENDING":
                 if self.timeout and (time.time() - start > self.timeout):
@@ -732,10 +828,14 @@ class Client:
                 if res is not None:
                     if not retriable(res.status_code, res.reason):
                         return res
+                    try:
+                        text = res.json()['detail']
+                    except requests.exceptions.JSONDecodeError:
+                        text = res.text
                     self.warning(
                         "Recovering from HTTP error [%s %s], attemps %s of %s",
                         res.status_code,
-                        res.json()["detail"],
+                        text,
                         tries,
                         self.retry_max,
                     )

ddsapi-0.6b3/ddsapi/cache.py

@@ -0,0 +1,107 @@
+"""This module contains simple query-hash-based cache mechanism."""
+
+import os
+import json
+from typing import Mapping, Optional, Iterable, Union
+from uuid import uuid4
+from dataclasses import dataclass, field
+import pickle
+import shutil
+
+import xarray as xr
+import hashlib
+
+
+@dataclass
+class CacheManager:
+    cache_dir: str = field(kw_only=True)
+    disabled: bool = field(default=False, init=True, kw_only=True)
+    cache_config: str = field(default=".config", init=False)
+    _cache: Mapping[str, str] = field(init=False)
+
+    def __post_init__(self):
+        self._init()
+        self._cache = self._read_cache() if not self.disabled else {}
+
+    def _init(self) -> None:
+        self._cache = {}
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+        if os.path.exists(os.path.join(self.cache_dir, self.cache_config)):
+            return
+        self._update_cache_config()
+
+    def _update_cache_config(self) -> None:
+        with open(os.path.join(self.cache_dir, self.cache_config), "wb") as file:
+            pickle.dump(self._cache, file=file)
+
+    def _read_cache(self) -> dict:
+        with open(os.path.join(self.cache_dir, self.cache_config), "rb") as file:
+            return pickle.load(file)
+
+    def _compute_hash(
+        self, dataset_id: str, product_id: str, request: dict
+    ) -> int:
+        return hashlib.sha256(
+            json.dumps(
+                dict(**{"dataset_id": dataset_id, "product_id": product_id}, **request)
+            ).encode()
+        ).hexdigest()
+
+    def _maybe_get_file_from_cache(
+        self, dataset_id: str, product_id: str, request: dict
+    ) -> Optional[str]:
+        query_hash: int = self._compute_hash(dataset_id, product_id, request)
+        return self._cache.get(query_hash)
+
+    def add_to_cache(
+        self,
+        dataset_id: str,
+        product_id: str,
+        request: dict,
+        target: Union[Iterable[str], str],
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        if self.disabled:
+            return
+        query_hash: int = self._compute_hash(dataset_id, product_id, request)
+        if query_hash in self._cache and not overwrite:
+            return
+        if isinstance(target, list):
+            cache_files = []
+            for f in target:
+                _, ext = os.path.splitext(f)
+                res_file = f"{uuid4()}{ext}"
+                cache_files.append(res_file)
+                shutil.move(f, os.path.join(self.cache_dir, res_file))
+            self._cache[query_hash] = cache_files
+        elif isinstance(target, str):
+            _, ext = os.path.splitext(target)
+            res_file = f"{uuid4()}{ext}"
+            shutil.move(target, os.path.join(self.cache_dir, res_file))
+            self._cache[query_hash] = res_file
+        self._update_cache_config()
+
+    def maybe_get_from_cache(
+        self, dataset_id: str, product_id: str, request: dict
+    ) -> Union[xr.Dataset, Iterable[xr.Dataset], None]:
+        target: Union[str, Iterable[str]] = self._maybe_get_file_from_cache(
+            dataset_id=dataset_id, product_id=product_id, request=request
+        )
+        if self.disabled:
+            return        
+        if not target:
+            return
+        if isinstance(target, list):
+            ds_list = []
+            for f in target:
+                ds = xr.open_dataset(os.path.join(self.cache_dir, f))
+                ds_list.append(ds)
+            if len(ds_list) == 1:
+                return ds_list[0]
+            return ds_list
+        elif isinstance(target, str):
+            return xr.open_dataset(os.path.join(self.cache_dir, target))
+        else:
+            raise TypeError

{ddsapi-0.6b0 → ddsapi-0.6b3}/ddsapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddsapi
-Version: 0.6b0
+Version: 0.6b3
 Summary: Python Client to access and download data from CMCC Data Delivery System (DDS)
 Home-page: https://github.com/CMCC-Foundation/ddsapi-client/
 Author: CMCC Data Delivery System Team
@@ -18,6 +18,10 @@ Classifier: Topic :: Scientific/Engineering :: Hydrology
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: netCDF4>=1.5.3
+Requires-Dist: scipy>=1.5.2
+Requires-Dist: requests>=2.23.0
+Requires-Dist: xarray>=0.16.0
 
 # DDSAPI-Client
 Python Client to access and download data from [CMCC Data Delivery System (DDS)](https://dds.cmcc.it)
@@ -40,6 +44,6 @@ $ pip install ddsapi
 To use the tool a file `$HOME/.ddsapirc` must be created as following
 
 ```bash
-url: https://ddsapi.cmcc.it/v1
+url: https://ddshub.cmcc.it/api/v2
 key: <api-key>
 ```

{ddsapi-0.6b0 → ddsapi-0.6b3}/ddsapi.egg-info/SOURCES.txt

@@ -3,8 +3,11 @@ README.md
 setup.py
 ddsapi/__init__.py
 ddsapi/api.py
+ddsapi/cache.py
 ddsapi.egg-info/PKG-INFO
 ddsapi.egg-info/SOURCES.txt
 ddsapi.egg-info/dependency_links.txt
 ddsapi.egg-info/requires.txt
-ddsapi.egg-info/top_level.txt
+ddsapi.egg-info/top_level.txt
+tests/__init__.py
+tests/test_config.py

{ddsapi-0.6b0 → ddsapi-0.6b3}/ddsapi.egg-info/requires.txt

@@ -1,4 +1,4 @@
 netCDF4>=1.5.3
 scipy>=1.5.2
 requests>=2.23.0
-xarray==0.16.0
+xarray>=0.16.0

{ddsapi-0.6b0 → ddsapi-0.6b3}/ddsapi.egg-info/top_level.txt

@@ -1 +1,2 @@
 ddsapi
+tests

{ddsapi-0.6b0 → ddsapi-0.6b3}/setup.py

@@ -17,7 +17,7 @@ with open("README.md", "r") as f:
 
 setuptools.setup(
     name="ddsapi",
-    version="0.6b0",
+    version="0.6b3",
     author="CMCC Data Delivery System Team",
     author_email="dds-support@cmcc.it",
     description=(
@@ -32,7 +32,7 @@ setuptools.setup(
         "netCDF4>=1.5.3",
         "scipy>=1.5.2",
         "requests>=2.23.0",
-        "xarray==0.16.0",
+        "xarray>=0.16.0",
     ],
     classifiers=[
         "Development Status :: 4 - Beta",

ddsapi-0.6b3/tests/test_config.py

@@ -0,0 +1,88 @@
+import os
+
+import pytest
+from unittest import mock
+
+from ddsapi.api import Config, EnvVarNames
+
+
+def _read_config_mock(*ar) -> dict:
+    return {
+        "url": "url_from_file",
+        "key": "key_from_file",
+        "verify": "0",
+        "cachedir": "cachedir_from_file",
+    }
+
+
+def _read_config_mock2(*ar) -> dict:
+    return {
+        "url": "url_from_file",
+        "key": "key_from_file",
+        "verify": "0",
+    }
+
+
+@mock.patch.dict(os.environ, {EnvVarNames.URL: "dds-url"})
+@mock.patch("ddsapi.api.read_config", return_value={})
+def test_get_url_from_env_var(read_config):
+    conf = Config()
+    assert conf.url == "dds-url"
+    assert conf.key is None
+    assert conf.cachedir == os.path.expanduser(
+        os.path.join("~", Config._CACHE_FILENAME)
+    )
+    assert conf.verify == 1
+
+
+@mock.patch.dict(os.environ, {EnvVarNames.KEY: "dds-key"})
+@mock.patch("ddsapi.api.read_config", return_value={})
+def test_get_key_from_env_var(read_config):
+    conf = Config()
+    assert conf.key == "dds-key"
+    assert conf.url is None
+    assert conf.cachedir == os.path.expanduser(
+        os.path.join("~", Config._CACHE_FILENAME)
+    )
+    assert conf.verify == 1
+
+
+@mock.patch.dict(os.environ, {EnvVarNames.CACHEDIR: "dds-cache"})
+@mock.patch("ddsapi.api.read_config", return_value={})
+def test_get_cache_from_env_var(read_config):
+    conf = Config()
+    assert conf.cachedir == "dds-cache"
+    assert conf.key is None
+    assert conf.url is None
+    assert conf.verify == 1
+
+
+@mock.patch("ddsapi.api.read_config", _read_config_mock)
+def test_read_conf_from_rc_file():
+    conf = Config()
+    assert conf.key == "key_from_file"
+    assert conf.url == "url_from_file"
+    assert conf.cachedir == "cachedir_from_file"
+    assert conf.verify == 0
+
+
+@mock.patch.dict(os.environ, {EnvVarNames.CACHEDIR: "dds-cache"})
+@mock.patch("ddsapi.api.read_config", _read_config_mock2)
+def test_conf_from_file_overwrites():
+    conf = Config()
+    assert conf.key == "key_from_file"
+    assert conf.url == "url_from_file"
+    assert conf.cachedir == "dds-cache"
+    assert conf.verify == 0
+
+
+@mock.patch.dict(os.environ, {EnvVarNames.KEY: "key0"})
+@mock.patch("ddsapi.api.read_config", return_value={"key": "key1"})
+def test_user_defined_overwrites_all(read_config):
+    conf = Config(key="key2")
+    assert conf.key == "key2"
+    assert conf.url is None
+    assert conf.cachedir == os.path.expanduser(
+        os.path.join("~", Config._CACHE_FILENAME)
+    )
+    assert conf.verify == 1