eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

eodag/utils/constraints.py

@@ -19,13 +19,14 @@ import copy
 import logging
 from typing import Any, Dict, List, Set, Union
 
-import requests
+from requests.auth import AuthBase
 
 from eodag.api.product.metadata_mapping import get_provider_queryable_key
 from eodag.plugins.apis.base import Api
 from eodag.plugins.search.base import Search
-from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, deepcopy
-from eodag.utils.exceptions import TimeOutError, ValidationError
+from eodag.utils import deepcopy
+from eodag.utils.exceptions import RequestError, ValidationError
+from eodag.utils.requests import fetch_json
 
 logger = logging.getLogger("eodag.constraints")
 
@@ -73,8 +74,20 @@ def get_constraint_queryables_with_additional_params(
             if provider_key and provider_key in constraint:
                 eodag_provider_key_mapping[provider_key] = param
                 params_available[param] = True
-                if value in constraint[provider_key]:
+                if (
+                    isinstance(value, list)
+                    and all([v in constraint[provider_key] for v in value])
+                    or not isinstance(value, list)
+                    and value in constraint[provider_key]
+                ):
                     params_matched[param] = True
+                elif isinstance(value, str):
+                    # for Copernicus providers, values can be multiple and represented with a string
+                    # separated by slashes (example: time = "0000/0100/0200")
+                    values = value.split("/")
+                    params_matched[param] = all(
+                        [v in constraint[provider_key] for v in values]
+                    )
                 values_available[param].update(constraint[provider_key])
         # match with default values of params
         for default_param, default_value in defaults.items():
@@ -167,42 +180,29 @@ def fetch_constraints(
     :returns: list of constraints fetched from the provider
     :rtype: List[Dict[Any, Any]]
     """
+    auth = (
+        plugin.auth
+        if hasattr(plugin, "auth") and isinstance(plugin.auth, AuthBase)
+        else None
+    )
     try:
-        headers = USER_AGENT
-        logger.debug("fetching constraints from %s", constraints_url)
-        if hasattr(plugin, "auth"):
-            res = requests.get(
-                constraints_url,
-                headers=headers,
-                auth=plugin.auth,
-                timeout=HTTP_REQ_TIMEOUT,
-            )
-        else:
-            res = requests.get(
-                constraints_url, headers=headers, timeout=HTTP_REQ_TIMEOUT
-            )
-        res.raise_for_status()
-    except requests.exceptions.Timeout as exc:
-        raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
-    except requests.exceptions.HTTPError as err:
-        logger.error(
-            "constraints could not be fetched from %s, error: %s",
-            constraints_url,
-            str(err),
-        )
+        constraints_data = fetch_json(constraints_url, auth=auth)
+    except RequestError as err:
+        logger.error(str(err))
+        return []
+
+    config = plugin.config.__dict__
+    if (
+        "constraints_entry" in config
+        and config["constraints_entry"]
+        and config["constraints_entry"] in constraints_data
+    ):
+        constraints = constraints_data[config["constraints_entry"]]
+    elif config.get("stop_without_constraints_entry_key", False):
         return []
     else:
-        constraints_data = res.json()
-        config = plugin.config.__dict__
-        if (
-            "constraints_entry" in config
-            and config["constraints_entry"]
-            and config["constraints_entry"] in constraints_data
-        ):
-            constraints = constraints_data[config["constraints_entry"]]
-        else:
-            constraints = constraints_data
-        return constraints
+        constraints = constraints_data
+    return constraints
 
 
 def _get_other_possible_values_for_values_with_defaults(

eodag/utils/exceptions.py

@@ -23,77 +23,83 @@ if TYPE_CHECKING:
     from typing import Optional, Set, Tuple
 
 
-class ValidationError(Exception):
+class EodagError(Exception):
+    """General EODAG error"""
+
+
+class ValidationError(EodagError):
     """Error validating data"""
 
-    def __init__(self, message: str) -> None:
+    def __init__(self, message: str, parameters: Set[str] = set()) -> None:
         self.message = message
+        self.parameters = parameters
 
 
-class PluginNotFoundError(Exception):
+class PluginNotFoundError(EodagError):
     """Error when looking for a plugin class that was not defined"""
 
 
-class PluginImplementationError(Exception):
+class PluginImplementationError(EodagError):
     """Error when a plugin does not behave as expected"""
 
 
-class MisconfiguredError(Exception):
+class MisconfiguredError(EodagError):
     """An error indicating a Search Plugin that is not well configured"""
 
 
-class AddressNotFound(Exception):
+class AddressNotFound(EodagError):
     """An error indicating the address of a subdataset was not found"""
 
 
-class UnsupportedProvider(Exception):
+class UnsupportedProvider(EodagError):
     """An error indicating that eodag does not support a provider"""
 
 
-class UnsupportedProductType(Exception):
+class UnsupportedProductType(EodagError):
     """An error indicating that eodag does not support a product type"""
 
     def __init__(self, product_type: str) -> None:
         self.product_type = product_type
 
 
-class UnsupportedDatasetAddressScheme(Exception):
+class UnsupportedDatasetAddressScheme(EodagError):
     """An error indicating that eodag does not yet support an address scheme for
     accessing raster subdatasets"""
 
 
-class AuthenticationError(Exception):
+class AuthenticationError(EodagError):
     """An error indicating that an authentication plugin did not succeeded
     authenticating a user"""
 
 
-class DownloadError(Exception):
+class DownloadError(EodagError):
     """An error indicating something wrong with the download process"""
 
 
-class NotAvailableError(Exception):
+class NotAvailableError(EodagError):
     """An error indicating that the product is not available for download"""
 
 
-class RequestError(Exception):
+class RequestError(EodagError):
     """An error indicating that a request has failed. Usually eodag functions
     and methods should catch and skip this"""
 
-    history: Set[Tuple[Exception, str]] = set()
+    history: Set[Tuple[str, Exception]] = set()
+    parameters: Set[str] = set()
 
     def __str__(self):
         repr = super().__str__()
         for err_tuple in self.history:
-            repr += f"\n- {str(err_tuple)}"
+            repr += f"- {str(err_tuple)}"
         return repr
 
 
-class NoMatchingProductType(Exception):
+class NoMatchingProductType(EodagError):
     """An error indicating that eodag was unable to derive a product type from a set
     of search parameters"""
 
 
-class STACOpenerError(Exception):
+class STACOpenerError(EodagError):
     """An error indicating that a STAC file could not be opened"""
 
 

eodag/utils/requests.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Optional, Tuple
+
+import requests
+
+from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, path_to_uri, uri_to_path
+from eodag.utils.exceptions import RequestError, TimeOutError
+
+logger = logging.getLogger("eodag.utils.requests")
+
+
+def fetch_json(
+    file_url: str,
+    req_session: Optional[requests.Session] = None,
+    auth: Optional[requests.AuthBase] = None,
+    timeout: float = HTTP_REQ_TIMEOUT,
+) -> Any:
+    """
+    Fetches http/distant or local json file
+
+    :param file_url: url from which the file can be fetched
+    :type file_url: str
+    :param req_session: (optional) requests session
+    :type req_session: requests.Session
+    :param auth: (optional) authenticated object if request needs authentication
+    :type auth: Optional[requests.AuthBase]
+    :param timeout: (optional) authenticated object
+    :type timeout: float
+    :returns: json file content
+    :rtype: Any
+    """
+    if req_session is None:
+        req_session = requests.Session()
+    try:
+        if not file_url.lower().startswith("http"):
+            file_url = path_to_uri(os.path.abspath(file_url))
+            req_session.mount("file://", LocalFileAdapter())
+
+        headers = USER_AGENT
+        logger.debug(f"fetching {file_url}")
+        res = req_session.get(
+            file_url,
+            headers=headers,
+            auth=auth,
+            timeout=timeout,
+        )
+        res.raise_for_status()
+    except requests.exceptions.Timeout as exc:
+        raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
+    except requests.exceptions.RequestException as exc:
+        raise RequestError(
+            f"Unable to fetch {file_url}: {str(exc)}",
+        ) from exc
+    else:
+        return res.json()
+
+
+class LocalFileAdapter(requests.adapters.BaseAdapter):
+    """Protocol Adapter to allow Requests to GET file:// URLs inspired
+    by https://stackoverflow.com/questions/10123929/fetch-a-file-from-a-local-url-with-python-requests/27786580
+    `LocalFileAdapter` class available for the moment (on the 2024-04-22)
+    """
+
+    @staticmethod
+    def _chkpath(method: str, path: str) -> Tuple[int, str]:
+        """Return an HTTP status for the given filesystem path.
+
+        :param method: method of the request
+        :type method: str
+        :param path: path of the given file
+        :type path: str
+        :returns: HTTP status and its associated message
+        :rtype: Tuple[int, str]
+        """
+        if method.lower() in ("put", "delete"):
+            return 501, "Not Implemented"  # TODO
+        elif method.lower() not in ("get", "head"):
+            return 405, "Method Not Allowed"
+        elif os.path.isdir(path):
+            return 400, "Path Not A File"
+        elif not os.path.isfile(path):
+            return 404, "File Not Found"
+        elif not os.access(path, os.R_OK):
+            return 403, "Access Denied"
+        else:
+            return 200, "OK"
+
+    def send(self, req: requests.PreparedRequest, **kwargs: Any) -> requests.Response:
+        """Wraps a file, described in request, in a Response object.
+
+        :param req: The PreparedRequest being "sent".
+        :type req: :class:`~requests.PreparedRequest`
+        :param kwargs: (not used) additionnal arguments of the request
+        :type kwargs: Any
+        :returns: a Response object containing the file
+        :rtype: :class:`~requests.Response`
+        """
+        response = requests.Response()
+
+        path_url = uri_to_path(req.url)
+
+        if req.method is None or req.url is None:
+            raise RequestError("Method or url of the request is missing")
+        response.status_code, response.reason = self._chkpath(req.method, path_url)
+        if response.status_code == 200 and req.method.lower() != "head":
+            try:
+                response.raw = open(path_url, "rb")
+            except (OSError, IOError) as err:
+                response.status_code = 500
+                response.reason = str(err)
+        response.url = req.url
+        response.request = req
+
+        return response
+
+    def close(self):
+        """Closes without cleaning up adapter specific items."""
+        pass

eodag/utils/rest.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""eodag.rest.utils methods that must be importable without eodag[server] installed"""
+
+from __future__ import annotations
+
+import datetime
+import re
+from typing import Any, Dict, Optional, Tuple
+
+import dateutil.parser
+from dateutil import tz
+
+from eodag.utils.exceptions import ValidationError
+
+RFC3339_PATTERN = (
+    r"^(\d{4})-(\d{2})-(\d{2})"
+    r"(?:T(\d{2}):(\d{2}):(\d{2})(\.\d+)?"
+    r"(Z|([+-])(\d{2}):(\d{2}))?)?$"
+)
+
+
+def get_datetime(arguments: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
+    """Get start and end dates from a dict containing `/` separated dates in `datetime` item
+
+    :param arguments: dict containing a single date or `/` separated dates in `datetime` item
+    :type arguments: dict
+    :returns: Start date and end date from datetime string (duplicate value if only one date as input)
+    :rtype: Tuple[Optional[str], Optional[str]]
+    """
+    datetime_str = arguments.pop("datetime", None)
+
+    if datetime_str:
+        datetime_split = datetime_str.split("/")
+        if len(datetime_split) > 1:
+            dtstart = datetime_split[0] if datetime_split[0] != ".." else None
+            dtend = datetime_split[1] if datetime_split[1] != ".." else None
+        elif len(datetime_split) == 1:
+            # same time for start & end if only one is given
+            dtstart, dtend = datetime_split[0:1] * 2
+        else:
+            return None, None
+
+        return get_date(dtstart), get_date(dtend)
+
+    else:
+        # return already set (dtstart, dtend) or None
+        dtstart = get_date(arguments.pop("dtstart", None))
+        dtend = get_date(arguments.pop("dtend", None))
+        return get_date(dtstart), get_date(dtend)
+
+
+def get_date(date: Optional[str]) -> Optional[str]:
+    """Check if the input date can be parsed as a date"""
+
+    if not date:
+        return None
+    try:
+        return (
+            dateutil.parser.parse(date)
+            .replace(tzinfo=tz.UTC)
+            .isoformat()
+            .replace("+00:00", "")
+        )
+    except ValueError as e:
+        exc = ValidationError("invalid input date: %s" % e)
+        raise exc
+
+
+def rfc3339_str_to_datetime(s: str) -> datetime.datetime:
+    """Convert a string conforming to RFC 3339 to a :class:`datetime.datetime`.
+
+    :param s: The string to convert to :class:`datetime.datetime`
+    :type s: str
+
+    :returns: The datetime represented by the ISO8601 (RFC 3339) formatted string
+    :rtype: :class:`datetime.datetime`
+
+    raises: :class:`ValidationError`
+    """
+    # Uppercase the string
+    s = s.upper()
+
+    # Match against RFC3339 regex.
+    result = re.match(RFC3339_PATTERN, s)
+    if not result:
+        raise ValidationError("Invalid RFC3339 datetime.")
+
+    return dateutil.parser.isoparse(s).replace(tzinfo=datetime.timezone.utc)

eodag/utils/stac_reader.py

@@ -27,8 +27,9 @@ from urllib.request import urlopen
 import concurrent.futures
 import orjson
 import pystac
+from pystac.stac_object import STACObjectType
 
-from eodag.utils import HTTP_REQ_TIMEOUT
+from eodag.utils import HTTP_REQ_TIMEOUT, get_ssl_context
 from eodag.utils.exceptions import STACOpenerError
 
 logger = logging.getLogger("eodag.utils.stac_reader")
@@ -38,10 +39,11 @@ class _TextOpener:
     """Exhaust read methods for pystac.StacIO in the order defined
     in the openers list"""
 
-    def __init__(self, timeout: int) -> None:
+    def __init__(self, timeout: int, ssl_verify: bool) -> None:
         self.openers = [self.read_local_json, self.read_http_remote_json]
         # Only used by read_http_remote_json
         self.timeout = timeout
+        self.ssl_verify = ssl_verify
 
     @staticmethod
     def read_local_json(url: str, as_json: bool = False) -> Any:
@@ -54,13 +56,14 @@ class _TextOpener:
                 with open(url) as f:
                     return f.read()
         except OSError:
-            logger.debug("read_local_json is not the right STAC opener")
-            raise STACOpenerError
+            raise STACOpenerError("read_local_json is not the right STAC opener")
 
     def read_http_remote_json(self, url: str, as_json: bool = False) -> Any:
         """Read JSON remote HTTP file"""
+        ssl_ctx = get_ssl_context(self.ssl_verify)
+
         try:
-            res = urlopen(url, timeout=self.timeout)
+            res = urlopen(url, timeout=self.timeout, context=ssl_ctx)
             content_type = res.getheader("Content-Type")
             if content_type is None:
                 encoding = "utf-8"
@@ -79,17 +82,19 @@ class _TextOpener:
                     f"{url} with a timeout of {self.timeout} seconds"
                 ) from None
             else:
-                logger.debug("read_http_remote_json is not the right STAC opener")
-                raise STACOpenerError
+                raise STACOpenerError(
+                    "read_http_remote_json is not the right STAC opener"
+                )
 
     def __call__(self, url: str, as_json: bool = False) -> Any:
+        openers = self.openers[:]
         res = None
-        while self.openers:
+        while openers:
             try:
-                res = self.openers[0](url, as_json)
+                res = openers[0](url, as_json)
             except STACOpenerError:
                 # Remove the opener that just failed
-                self.openers.pop(0)
+                openers.pop(0)
             if res is not None:
                 break
         if res is None:
@@ -102,6 +107,7 @@ def fetch_stac_items(
     recursive: bool = False,
     max_connections: int = 100,
     timeout: int = HTTP_REQ_TIMEOUT,
+    ssl_verify: bool = True,
 ) -> List[Dict[str, Any]]:
     """Fetch STAC item from a single item file or items from a catalog.
 
@@ -112,14 +118,16 @@ def fetch_stac_items(
     :param max_connections: (optional) Maximum number of connections for HTTP requests
     :type max_connections: int
     :param timeout: (optional) Timeout in seconds for each internal HTTP request
-    :type timeout: float
+    :type timeout: int
+    :param ssl_verify: (optional) SSL Verification for HTTP request
+    :type ssl_verify: bool
     :returns: The items found in `stac_path`
     :rtype: :class:`list`
     """
 
     # URI opener used by PySTAC internally, instantiated here
     # to retrieve the timeout.
-    _text_opener = _TextOpener(timeout)
+    _text_opener = _TextOpener(timeout, ssl_verify)
     pystac.StacIO.read_text = _text_opener
 
     stac_obj = pystac.read_file(stac_path)
@@ -142,6 +150,8 @@ def _fetch_stac_items_from_catalog(
     _text_opener: Callable[[str, bool], Any],
 ) -> List[Any]:
     """Fetch items from a STAC catalog"""
+    items: List[Dict[Any, Any]] = []
+
     # pystac cannot yet return links from a single file catalog, see:
     # https://github.com/stac-utils/pystac/issues/256
     extensions: Optional[Union[List[str], str]] = getattr(cat, "stac_extensions", None)
@@ -151,8 +161,7 @@ def _fetch_stac_items_from_catalog(
             items = [feature for feature in cat.to_dict()["features"]]
             return items
 
-    # Making the links absolutes allow for both relative and absolute links
-    # to be handled.
+    # Making the links absolutes allow for both relative and absolute links to be handled.
     if not recursive:
         hrefs: List[Optional[str]] = [
             link.get_absolute_href() for link in cat.get_item_links()
@@ -164,7 +173,6 @@ def _fetch_stac_items_from_catalog(
                 link.get_absolute_href() for link in parent_catalog.get_item_links()
             ]
 
-    items: List[Dict[Any, Any]] = []
     if hrefs:
         logger.debug("Fetching %s items", len(hrefs))
         with concurrent.futures.ThreadPoolExecutor(
@@ -176,5 +184,81 @@ def _fetch_stac_items_from_catalog(
             for future in concurrent.futures.as_completed(future_to_href):
                 item = future.result()
                 if item:
-                    items.append(future.result())
+                    items.append(item)
     return items
+
+
+def fetch_stac_collections(
+    stac_path: str,
+    collection: Optional[str] = None,
+    max_connections: int = 100,
+    timeout: int = HTTP_REQ_TIMEOUT,
+    ssl_verify: bool = True,
+) -> List[Dict[str, Any]]:
+    """Fetch STAC collection(s) from a catalog.
+
+    :param stac_path: A STAC object filepath
+    :type stac_path: str
+    :param collection: the collection to fetch
+    :type collection: Optional[str]
+    :param max_connections: (optional) Maximum number of connections for HTTP requests
+    :type max_connections: int
+    :param timeout: (optional) Timeout in seconds for each internal HTTP request
+    :type timeout: int
+    :param ssl_verify: (optional) SSL Verification for HTTP request
+    :type ssl_verify: bool
+    :returns: The collection(s) found in `stac_path`
+    :rtype: :class:`list`
+    """
+
+    # URI opener used by PySTAC internally, instantiated here to retrieve the timeout.
+    _text_opener = _TextOpener(timeout, ssl_verify)
+    pystac.StacIO.read_text = _text_opener
+
+    stac_obj = pystac.read_file(stac_path)
+    if isinstance(stac_obj, pystac.Catalog):
+        return _fetch_stac_collections_from_catalog(
+            stac_obj, collection, max_connections, _text_opener
+        )
+    else:
+        raise STACOpenerError(f"{stac_path} must be a STAC catalog")
+
+
+def _fetch_stac_collections_from_catalog(
+    cat: pystac.Catalog,
+    collection: Optional[str],
+    max_connections: int,
+    _text_opener: Callable[[str, bool], Any],
+) -> List[Any]:
+    """Fetch collections from a STAC catalog"""
+    collections: List[Dict[Any, Any]] = []
+
+    # Making the links absolutes allow for both relative and absolute links to be handled.
+    hrefs: List[Optional[str]] = [
+        link.get_absolute_href()
+        for link in cat.get_child_links()
+        if collection is not None and link.title == collection
+    ]
+    if len(hrefs) == 0:
+        hrefs = [link.get_absolute_href() for link in cat.get_child_links()]
+
+    if hrefs:
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=max_connections
+        ) as executor:
+            future_to_href = (
+                executor.submit(_text_opener, str(href), True) for href in hrefs
+            )
+            for future in concurrent.futures.as_completed(future_to_href):
+                fetched_collection = future.result()
+                if (
+                    fetched_collection
+                    and fetched_collection["type"] == STACObjectType.COLLECTION
+                    and (
+                        collection is None
+                        or collection is not None
+                        and fetched_collection.get("id") == collection
+                    )
+                ):
+                    collections.append(fetched_collection)
+    return collections