ecmwf-datastores-client 0.1.0__py3-none-any.whl

ecmwf/datastores/__init__.py

@@ -0,0 +1,39 @@
+"""ECMWF Data Stores Service (DSS) API Python client."""
+
+# Copyright 2022, European Union.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+    # NOTE: the `version.py` file must not be present in the git repository
+    #   as it is generated by setuptools at install time
+    from ecmwf.datastores.version import __version__
+except ImportError:  # pragma: no cover
+    # Local copy or not installed with setuptools
+    __version__ = "999"
+
+from .catalogue import Collection, Collections
+from .client import Client
+from .processing import Jobs, Process, Processes, Remote, Results
+
+__all__ = [
+    "__version__",
+    "Client",
+    "Collection",
+    "Collections",
+    "Jobs",
+    "Process",
+    "Processes",
+    "Remote",
+    "Results",
+]

ecmwf/datastores/catalogue.py

@@ -0,0 +1,206 @@
+# Copyright 2022, European Union.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import datetime
+import warnings
+from typing import Any, Callable
+
+import attrs
+import requests
+
+from ecmwf import datastores
+from ecmwf.datastores import config, utils
+from ecmwf.datastores.processing import ApiResponse, ApiResponsePaginated, RequestKwargs
+
+
+@attrs.define
+class Collections(ApiResponsePaginated):
+    """A class to interact with catalogue collections."""
+
+    @property
+    def collection_ids(self) -> list[str]:
+        """List of collection IDs."""
+        return [collection["id"] for collection in self._json_dict["collections"]]
+
+
+@attrs.define
+class Collection(ApiResponse):
+    """A class to interact with a catalogue collection."""
+
+    @property
+    def begin_datetime(self) -> datetime.datetime | None:
+        """Begin datetime of the collection."""
+        if (value := self._json_dict["extent"]["temporal"]["interval"][0][0]) is None:
+            return value
+        return utils.string_to_datetime(value)
+
+    @property
+    def end_datetime(self) -> datetime.datetime | None:
+        """End datetime of the collection."""
+        if (value := self._json_dict["extent"]["temporal"]["interval"][0][1]) is None:
+            return value
+        return utils.string_to_datetime(value)
+
+    @property
+    def published_at(self) -> datetime.datetime:
+        """When the collection was first published."""
+        return utils.string_to_datetime(self._json_dict["published"])
+
+    @property
+    def updated_at(self) -> datetime.datetime:
+        """When the collection was last updated."""
+        return utils.string_to_datetime(self._json_dict["updated"])
+
+    @property
+    def title(self) -> str:
+        """Title of the collection."""
+        value = self._json_dict["title"]
+        assert isinstance(value, str)
+        return value
+
+    @property
+    def description(self) -> str:
+        """Description of the collection."""
+        value = self._json_dict["description"]
+        assert isinstance(value, str)
+        return value
+
+    @property
+    def bbox(self) -> tuple[float, float, float, float]:
+        """Bounding box of the collection (W, S, E, N)."""
+        return tuple(self._json_dict["extent"]["spatial"]["bbox"][0])
+
+    @property
+    def id(self) -> str:
+        """Collection ID."""
+        return str(self._json_dict["id"])
+
+    @property
+    def _process(self) -> datastores.Process:
+        url = self._get_link_href(rel="retrieve")
+        return datastores.Process.from_request("get", url, **self._request_kwargs)
+
+    @property
+    def process(self) -> datastores.Process:
+        warnings.warn(
+            "`process` has been deprecated, and in the future will raise an error."
+            "Please use `submit` and `apply_constraints` from now on.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._process
+
+    @property
+    def form(self) -> list[dict[str, Any]]:
+        url = f"{self.url}/form.json"
+        return ApiResponse.from_request(
+            "get", url, log_messages=False, **self._request_kwargs
+        )._json_list
+
+    @property
+    def constraints(self) -> list[dict[str, Any]]:
+        url = f"{self.url}/constraints.json"
+        return ApiResponse.from_request(
+            "get", url, log_messages=False, **self._request_kwargs
+        )._json_list
+
+    def submit(self, request: dict[str, Any]) -> datastores.Remote:
+        """Submit a request.
+
+        Parameters
+        ----------
+        request: dict[str,Any]
+            Request parameters.
+
+        Returns
+        -------
+        datastores.Remote
+        """
+        return self._process.submit(request)
+
+    def apply_constraints(self, request: dict[str, Any]) -> dict[str, Any]:
+        """Apply constraints to the parameters in a request.
+
+        Parameters
+        ----------
+        request: dict[str,Any]
+            Request parameters.
+
+        Returns
+        -------
+        dict[str,Any]
+            Dictionary of valid values.
+        """
+        return self._process.apply_constraints(request)
+
+    def estimate_costs(self, request: dict[str, Any]) -> dict[str, Any]:
+        return self._process.estimate_costs(request)
+
+
+@attrs.define(slots=False)
+class Catalogue:
+    url: str
+    headers: dict[str, Any]
+    session: requests.Session
+    retry_options: dict[str, Any]
+    request_options: dict[str, Any]
+    download_options: dict[str, Any]
+    sleep_max: float
+    cleanup: bool
+    log_callback: Callable[..., None] | None
+    force_exact_url: bool = False
+
+    def __attrs_post_init__(self) -> None:
+        if not self.force_exact_url:
+            self.url += f"/{config.SUPPORTED_API_VERSION}"
+
+    @property
+    def _request_kwargs(self) -> RequestKwargs:
+        return RequestKwargs(
+            headers=self.headers,
+            session=self.session,
+            retry_options=self.retry_options,
+            request_options=self.request_options,
+            download_options=self.download_options,
+            sleep_max=self.sleep_max,
+            cleanup=self.cleanup,
+            log_callback=self.log_callback,
+        )
+
+    def get_collections(self, search_stats: bool = False, **params: Any) -> Collections:
+        url = f"{self.url}/datasets"
+        params["search_stats"] = search_stats
+        return Collections.from_request(
+            "get", url, params=params, **self._request_kwargs
+        )
+
+    def get_collection(self, collection_id: str) -> Collection:
+        url = f"{self.url}/collections/{collection_id}"
+        return Collection.from_request("get", url, **self._request_kwargs)
+
+    def get_licenses(self, **params: Any) -> dict[str, Any]:
+        url = f"{self.url}/vocabularies/licences"
+        response = ApiResponse.from_request(
+            "get", url, params=params, **self._request_kwargs
+        )
+        return response._json_dict
+
+    @property
+    def messages(self) -> ApiResponse:
+        url = f"{self.url}/messages"
+        return ApiResponse.from_request(
+            "get", url, log_messages=False, **self._request_kwargs
+        )

ecmwf/datastores/client.py

@@ -0,0 +1,400 @@
+# Copyright 2022, European Union.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import functools
+import warnings
+from typing import Any, Callable, Literal
+
+import attrs
+import multiurl.base
+import requests
+
+from ecmwf import datastores
+from ecmwf.datastores import config
+from ecmwf.datastores.catalogue import Catalogue
+from ecmwf.datastores.processing import Processing, RequestKwargs
+from ecmwf.datastores.profile import Profile
+
+
+@attrs.define(slots=False)
+class Client:
+    """ECMWF Data Stores Service (DSS) API Python client.
+
+    Parameters
+    ----------
+    url: str or None, default: None
+        API URL. If None, infer from ECMWF_DATASTORES_URL or ECMWF_DATASTORES_RC_FILE.
+    key: str or None, default: None
+        API Key. If None, infer from ECMWF_DATASTORES_KEY or ECMWF_DATASTORES_RC_FILE.
+    verify: bool, default: True
+        Whether to verify the TLS certificate at the remote end.
+    timeout: float or tuple[float,float], default: 60
+        How many seconds to wait for the server to send data, as a float, or a (connect, read) tuple.
+    progress: bool, default: True
+        Whether to display the progress bar during download.
+    cleanup: bool, default: False
+        Whether to delete requests after completion.
+    sleep_max: float, default: 120
+        Maximum time to wait (in seconds) while checking for a status change.
+    retry_after: float, default: 120
+        Time to wait (in seconds) between retries.
+    maximum_tries: int, default: 500
+        Maximum number of retries.
+    session: requests.Session
+        Requests session.
+    """
+
+    url: str | None = None
+    key: str | None = None
+    verify: bool = True
+    timeout: float | tuple[float, float] = 60
+    progress: bool = True
+    cleanup: bool = False
+    sleep_max: float = 120
+    retry_after: float = 120
+    maximum_tries: int = 500
+    session: requests.Session = attrs.field(factory=requests.Session)
+    _log_callback: Callable[..., None] | None = None
+
+    def __attrs_post_init__(self) -> None:
+        if self.url is None:
+            self.url = str(config.get_config("url"))
+
+        if self.key is None:
+            try:
+                self.key = str(config.get_config("key"))
+            except (KeyError, FileNotFoundError):
+                warnings.warn("The API key is missing", UserWarning)
+
+        try:
+            self._catalogue_api.messages.log_messages()
+        except Exception as exc:
+            warnings.warn(str(exc), UserWarning)
+
+    def _get_headers(self, key_is_mandatory: bool = True) -> dict[str, str]:
+        headers = {"User-Agent": f"ecmwf-datastores-client/{datastores.__version__}"}
+        if self.key is not None:
+            headers["PRIVATE-TOKEN"] = self.key
+        elif key_is_mandatory:
+            raise ValueError("The API key is needed to access this resource")
+        return headers
+
+    @property
+    def _retry_options(self) -> dict[str, Any]:
+        return {
+            "maximum_tries": self.maximum_tries,
+            "retry_after": self.retry_after,
+        }
+
+    @property
+    def _download_options(self) -> dict[str, Any]:
+        progress_bar = (
+            multiurl.base.progress_bar if self.progress else multiurl.base.NoBar
+        )
+        return {
+            "progress_bar": progress_bar,
+        }
+
+    @property
+    def _request_options(self) -> dict[str, Any]:
+        return {
+            "timeout": self.timeout,
+            "verify": self.verify,
+        }
+
+    def _get_request_kwargs(self, key_is_mandatory: bool = True) -> RequestKwargs:
+        return RequestKwargs(
+            headers=self._get_headers(key_is_mandatory=key_is_mandatory),
+            session=self.session,
+            retry_options=self._retry_options,
+            request_options=self._request_options,
+            download_options=self._download_options,
+            sleep_max=self.sleep_max,
+            cleanup=self.cleanup,
+            log_callback=self._log_callback,
+        )
+
+    @functools.cached_property
+    def _catalogue_api(self) -> Catalogue:
+        return Catalogue(
+            f"{self.url}/catalogue",
+            **self._get_request_kwargs(key_is_mandatory=False),
+        )
+
+    @functools.cached_property
+    def _retrieve_api(self) -> Processing:
+        return Processing(f"{self.url}/retrieve", **self._get_request_kwargs())
+
+    @functools.cached_property
+    def _profile_api(self) -> Profile:
+        return Profile(f"{self.url}/profiles", **self._get_request_kwargs())
+
+    def accept_licence(self, licence_id: str, revision: int) -> dict[str, Any]:
+        return self._profile_api.accept_licence(licence_id, revision=revision)
+
+    def apply_constraints(
+        self, collection_id: str, request: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Apply constraints to the parameters in a request.
+
+        Parameters
+        ----------
+        collection_id: str
+            Collection ID (e.g., ``"projections-cmip6"``).
+        request: dict[str,Any]
+            Request parameters.
+
+        Returns
+        -------
+        dict[str,Any]
+            Dictionary of valid values.
+        """
+        return self.get_process(collection_id).apply_constraints(request)
+
+    def check_authentication(self) -> dict[str, Any]:
+        """Verify authentication.
+
+        Returns
+        -------
+        dict[str,Any]
+            Content of the response.
+
+        Raises
+        ------
+        requests.HTTPError
+            If the authentication fails.
+        """
+        return self._profile_api.check_authentication()
+
+    def download_results(self, request_id: str, target: str | None = None) -> str:
+        """Download the results of a request.
+
+        Parameters
+        ----------
+        request_id: str
+            Request ID.
+        target: str or None
+            Target path. If None, download to the working directory.
+
+        Returns
+        -------
+        str
+            Path to the retrieved file.
+        """
+        return self.get_remote(request_id).download(target)
+
+    def estimate_costs(self, collection_id: str, request: Any) -> dict[str, Any]:
+        return self.get_process(collection_id).estimate_costs(request)
+
+    def get_accepted_licences(
+        self,
+        scope: Literal[None, "all", "dataset", "portal"] = None,
+    ) -> list[dict[str, Any]]:
+        params = {k: v for k, v in zip(["scope"], [scope]) if v is not None}
+        licences: list[dict[str, Any]]
+        licences = self._profile_api.accepted_licences(**params).get("licences", [])
+        return licences
+
+    def get_collection(self, collection_id: str) -> datastores.Collection:
+        """Retrieve a catalogue collection.
+
+        Parameters
+        ----------
+        collection_id: str
+            Collection ID (e.g., ``"projections-cmip6"``).
+
+        Returns
+        -------
+        datastores.Collection
+        """
+        return self._catalogue_api.get_collection(collection_id)
+
+    def get_collections(
+        self,
+        limit: int | None = None,
+        sortby: Literal[None, "id", "relevance", "title", "update"] = None,
+        query: str | None = None,
+        keywords: list[str] | None = None,
+    ) -> datastores.Collections:
+        """Retrieve catalogue collections.
+
+        Parameters
+        ----------
+        limit: int | None
+            Number of processes per page.
+        sortby: {None, 'id', 'relevance', 'title', 'update'}
+            Field to sort results by.
+        query: str or None
+            Full-text search query.
+        keywords: list[str] or None
+            Filter by keywords.
+
+        Returns
+        -------
+        datastores.Collections
+        """
+        params: dict[str, Any] = {
+            k: v
+            for k, v in zip(
+                ["limit", "sortby", "q", "kw"], [limit, sortby, query, keywords]
+            )
+            if v is not None
+        }
+        return self._catalogue_api.get_collections(**params)
+
+    def get_jobs(
+        self,
+        limit: int | None = None,
+        sortby: Literal[None, "created", "-created"] = None,
+        status: Literal[None, "accepted", "running", "successful", "failed"] = None,
+    ) -> datastores.Jobs:
+        """Retrieve submitted jobs.
+
+        Parameters
+        ----------
+        limit: int or None
+            Number of processes per page.
+        sortby: {None, 'created', '-created'}
+            Field to sort results by.
+        status: {None, 'accepted', 'running', 'successful', 'failed'}
+            Status of the results.
+
+        Returns
+        -------
+        datastores.Jobs
+        """
+        params = {
+            k: v
+            for k, v in zip(["limit", "sortby", "status"], [limit, sortby, status])
+            if v is not None
+        }
+        return self._retrieve_api.get_jobs(**params)
+
+    def get_licences(
+        self,
+        scope: Literal[None, "all", "dataset", "portal"] = None,
+    ) -> list[dict[str, Any]]:
+        params = {k: v for k, v in zip(["scope"], [scope]) if v is not None}
+        licences: list[dict[str, Any]]
+        licences = self._catalogue_api.get_licenses(**params).get("licences", [])
+        return licences
+
+    def get_process(self, collection_id: str) -> datastores.Process:
+        return self._retrieve_api.get_process(collection_id)
+
+    def get_processes(
+        self,
+        limit: int | None = None,
+        sortby: Literal[None, "id", "-id"] = None,
+    ) -> datastores.Processes:
+        params = {
+            k: v for k, v in zip(["limit", "sortby"], [limit, sortby]) if v is not None
+        }
+        return self._retrieve_api.get_processes(**params)
+
+    def get_remote(self, request_id: str) -> datastores.Remote:
+        """
+        Retrieve the remote object of a request.
+
+        Parameters
+        ----------
+        request_id: str
+            Request ID.
+
+        Returns
+        -------
+        datastores.Remote
+        """
+        return self._retrieve_api.get_job(request_id).get_remote()
+
+    def get_results(self, request_id: str) -> datastores.Results:
+        """
+        Retrieve the results of a request.
+
+        Parameters
+        ----------
+        request_id: str
+            Request ID.
+
+        Returns
+        -------
+        datastores.Results
+        """
+        return self.get_remote(request_id).get_results()
+
+    def retrieve(
+        self,
+        collection_id: str,
+        request: dict[str, Any],
+        target: str | None = None,
+    ) -> str:
+        """Submit a request and retrieve the results.
+
+        Parameters
+        ----------
+        collection_id: str
+            Collection ID (e.g., ``"projections-cmip6"``).
+        request: dict[str,Any]
+            Request parameters.
+        target: str or None
+            Target path. If None, download to the working directory.
+
+        Returns
+        -------
+        str
+            Path to the retrieved file.
+        """
+        return self.submit(collection_id, request).download(target)
+
+    def star_collection(self, collection_id: str) -> list[str]:
+        return self._profile_api.star_collection(collection_id)
+
+    def submit(self, collection_id: str, request: dict[str, Any]) -> datastores.Remote:
+        """Submit a request.
+
+        Parameters
+        ----------
+        collection_id: str
+            Collection ID (e.g., ``"projections-cmip6"``).
+        request: dict[str,Any]
+            Request parameters.
+
+        Returns
+        -------
+        datastores.Remote
+        """
+        return self._retrieve_api.submit(collection_id, request)
+
+    def submit_and_wait_on_results(
+        self, collection_id: str, request: dict[str, Any]
+    ) -> datastores.Results:
+        """Submit a request and wait for the results to be ready.
+
+        Parameters
+        ----------
+        collection_id: str
+            Collection ID (e.g., ``"projections-cmip6"``).
+        request: dict[str,Any]
+            Request parameters.
+
+        Returns
+        -------
+        datastores.Results
+        """
+        return self._retrieve_api.submit(collection_id, request).get_results()
+
+    def unstar_collection(self, collection_id: str) -> None:
+        return self._profile_api.unstar_collection(collection_id)

ecmwf/datastores/config.py

@@ -0,0 +1,45 @@
+# Copyright 2022, European Union.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+
+SUPPORTED_API_VERSION = "v1"
+CONFIG_PREFIX = "ecmwf_datastores"
+
+
+def read_config(path: str | None = None) -> dict[str, str]:
+    if path is None:
+        path = os.getenv(
+            f"{CONFIG_PREFIX}_RC_FILE".upper(),
+            f"~/.{CONFIG_PREFIX}rc".replace("_", "").lower(),
+        )
+    path = os.path.expanduser(path)
+    try:
+        config = {}
+        with open(path) as f:
+            for line in f.readlines():
+                if ":" in line:
+                    key, value = line.strip().split(":", 1)
+                    config[key] = value.strip()
+        return config
+    except FileNotFoundError:
+        raise
+    except Exception:
+        raise ValueError(f"Failed to parse {path!r} file")
+
+
+def get_config(key: str, config_path: str | None = None) -> str:
+    return os.getenv(f"{CONFIG_PREFIX}_{key}".upper()) or read_config(config_path)[key]