eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/authentication/token.py

@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import requests
@@ -28,7 +28,13 @@ from requests.auth import AuthBase
 from urllib3 import Retry
 
 from eodag.plugins.authentication.base import Authentication
-from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT
+from eodag.utils import (
+    HTTP_REQ_TIMEOUT,
+    REQ_RETRY_BACKOFF_FACTOR,
+    REQ_RETRY_STATUS_FORCELIST,
+    REQ_RETRY_TOTAL,
+    USER_AGENT,
+)
 from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, TimeOutError
 
 if TYPE_CHECKING:
@@ -41,7 +47,45 @@ logger = logging.getLogger("eodag.authentication.token")
 
 
 class TokenAuth(Authentication):
-    """TokenAuth authentication plugin"""
+    """TokenAuth authentication plugin - fetches a token which is added to search/download requests.
+
+    When using headers, if only :attr:`~eodag.config.PluginConfig.headers` is given, it will be used for both token
+    retrieve and authentication. If :attr:`~eodag.config.PluginConfig.retrieve_headers` is given, it will be used for
+    token retrieve only. If both are given, :attr:`~eodag.config.PluginConfig.retrieve_headers` will be used for token
+    retrieve and :attr:`~eodag.config.PluginConfig.headers` for authentication.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): TokenAuth
+        * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to fetch
+          the access token with user/password
+        * :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``): Dictionary containing all
+          keys/value pairs that should be added to the headers
+        * :attr:`~eodag.config.PluginConfig.retrieve_headers` (``dict[str, str]``): Dictionary containing all
+          keys/value pairs that should be added to the headers for token retrieve only
+        * :attr:`~eodag.config.PluginConfig.refresh_uri` (``str``) : url used to fetch the
+          access token with a refresh token
+        * :attr:`~eodag.config.PluginConfig.token_type` (``str``): type of the token (``json``
+          or ``text``); default: ``text``
+        * :attr:`~eodag.config.PluginConfig.token_key` (``str``): (mandatory if token_type=json)
+          key to get the access token in the response to the token request
+        * :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): key to get the refresh
+          token in the response to the token request
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
+          should be verified in the requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
+          returned in case of an authentication error
+        * :attr:`~eodag.config.PluginConfig.req_data` (``dict[str, Any]``): if the credentials
+          should be sent as data in the post request, the json structure can be given in this parameter
+        * :attr:`~eodag.config.PluginConfig.retry_total` (``int``): :class:`urllib3.util.Retry` ``total`` parameter,
+          total number of retries to allow; default: ``3``
+        * :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
+          ``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
+        * :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
+          ``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
+          ``[401, 429, 500, 502, 503, 504]``
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(TokenAuth, self).__init__(provider, config)
@@ -56,11 +100,29 @@ class TokenAuth(Authentication):
             self.config.auth_uri = self.config.auth_uri.format(
                 **self.config.credentials
             )
-            # format headers if needed (and accepts {token} to be formatted later)
+
+            # Format headers if needed with values from the credentials. Note:
+            # if only 'headers' is given, it will be used for both token retrieve and authentication.
+            # if 'retrieve_headers' is given, it will be used for token retrieve only.
+            # if both are given, 'retrieve_headers' will be used for token retrieve and 'headers' for authentication.
+
+            # If the authentication headers are undefined or None: use an empty dict.
+            # And don't format '{token}' now, it will be done later.
+            raw_headers = getattr(self.config, "headers", None) or {}
             self.config.headers = {
                 header: value.format(**{"token": "{token}", **self.config.credentials})
-                for header, value in getattr(self.config, "headers", {}).items()
+                for header, value in raw_headers.items()
             }
+
+            # If the retrieve headers are undefined, their attribute must not be set in self.config.
+            # If they are defined but empty, use an empty dict instead of None.
+            if hasattr(self.config, "retrieve_headers"):
+                raw_retrieve_headers = self.config.retrieve_headers or {}
+                self.config.retrieve_headers = {
+                    header: value.format(**self.config.credentials)
+                    for header, value in raw_retrieve_headers.items()
+                }
+
         except KeyError as e:
             raise MisconfiguredError(
                 f"Missing credentials inputs for provider {self.provider}: {e}"
@@ -89,14 +151,15 @@ class TokenAuth(Authentication):
                 and e.response.status_code in auth_errors
             ):
                 raise AuthenticationError(
-                    f"HTTP Error {e.response.status_code} returned, {response_text}\n"
-                    f"Please check your credentials for {self.provider}"
-                )
+                    f"Please check your credentials for {self.provider}.",
+                    f"HTTP Error {e.response.status_code} returned.",
+                    response_text,
+                ) from e
             # other error
             else:
                 raise AuthenticationError(
-                    f"Could no get authentication token: {str(e)}, {response_text}"
-                )
+                    "Could no get authentication token", str(e), response_text
+                ) from e
         else:
             if getattr(self.config, "token_type", "text") == "json":
                 token = response.json()[self.config.token_key]
@@ -116,16 +179,29 @@ class TokenAuth(Authentication):
         self,
         session: requests.Session,
     ) -> requests.Response:
+        retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
+        retry_backoff_factor = getattr(
+            self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
+        )
+        retry_status_forcelist = getattr(
+            self.config, "retry_status_forcelist", REQ_RETRY_STATUS_FORCELIST
+        )
+
         retries = Retry(
-            total=3,
-            backoff_factor=2,
-            status_forcelist=[401, 429, 500, 502, 503, 504],
+            total=retry_total,
+            backoff_factor=retry_backoff_factor,
+            status_forcelist=retry_status_forcelist,
         )
 
+        # Use the headers for retrieval if defined, else the headers for authentication
+        try:
+            headers = self.config.retrieve_headers
+        except AttributeError:
+            headers = self.config.headers
+
         # append headers to req if some are specified in config
-        req_kwargs: Dict[str, Any] = {
-            "headers": dict(self.config.headers, **USER_AGENT)
-        }
+        req_kwargs: dict[str, Any] = {"headers": dict(headers, **USER_AGENT)}
+        ssl_verify = getattr(self.config, "ssl_verify", True)
 
         if self.refresh_token:
             logger.debug("fetching access token with refresh token")
@@ -135,6 +211,7 @@ class TokenAuth(Authentication):
                     self.config.refresh_uri,
                     data={"refresh_token": self.refresh_token},
                     timeout=HTTP_REQ_TIMEOUT,
+                    verify=ssl_verify,
                     **req_kwargs,
                 )
                 response.raise_for_status()
@@ -170,6 +247,7 @@ class TokenAuth(Authentication):
             method=method,
             url=self.config.auth_uri,
             timeout=HTTP_REQ_TIMEOUT,
+            verify=ssl_verify,
             **req_kwargs,
         )
 
@@ -182,7 +260,7 @@ class RequestsTokenAuth(AuthBase):
         token: str,
         where: str,
         qs_key: Optional[str] = None,
-        headers: Optional[Dict[str, str]] = None,
+        headers: Optional[dict[str, str]] = None,
     ) -> None:
         self.token = token
         self.where = where

eodag/plugins/authentication/token_exchange.py

@@ -38,25 +38,23 @@ class OIDCTokenExchangeAuth(Authentication):
     """Token exchange implementation using
         :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject.
 
-    The configuration keys of this plugin are as follows (they have no defaults)::
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.subject` (``dict[str, Any]``) (**mandatory**):
+          The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin
+          configuration used to retrieve subject token
+        * :attr:`~eodag.config.PluginConfig.subject_issuer` (``str``) (**mandatory**): Identifies
+          the issuer of the subject_token
+        * :attr:`~eodag.config.PluginConfig.token_uri` (``str``) (**mandatory**): The url to
+          query to get the authorized token
+        * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): The OIDC
+          provider's client ID of the eodag provider
+        * :attr:`~eodag.config.PluginConfig.audience` (``str``) (**mandatory**): This parameter
+          specifies the target client you want the new token minted for.
+        * :attr:`~eodag.config.PluginConfig.token_key` (``str``) (**mandatory**): The key
+          pointing to the token in the json response to the POST request to the token server
 
-        # (mandatory) The full OIDCAuthorizationCodeFlowAuth plugin configuration used to retrieve subject token
-        subject:
-
-        # (mandatory) Identifies the issuer of the subject_token
-        subject_issuer:
-
-        # (mandatory) The url to query to get the authorized token
-        token_uri:
-
-        # (mandatory) The OIDC provider's client ID of the eodag provider
-        client_id:
-
-        # (mandatory) This parameter specifies the target client you want the new token minted for.
-        audience:
-
-        # (mandatory) The key pointing to the token in the json response to the POST request to the token server
-        token_key:
     """
 
     GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
@@ -71,7 +69,7 @@ class OIDCTokenExchangeAuth(Authentication):
     ]
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
-        super(OIDCTokenExchangeAuth, self).__init__(provider, config)
+        super().__init__(provider, config)
         for required_key in self.REQUIRED_KEYS:
             if getattr(self.config, required_key, None) is None:
                 raise MisconfiguredError(
@@ -100,12 +98,14 @@ class OIDCTokenExchangeAuth(Authentication):
             "audience": self.config.audience,
         }
         logger.debug("Getting target auth token")
+        ssl_verify = getattr(self.config, "ssl_verify", True)
         try:
             auth_response = self.subject.session.post(
                 self.config.token_uri,
                 data=auth_data,
                 headers=USER_AGENT,
                 timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             auth_response.raise_for_status()
         except requests.exceptions.Timeout as exc:

eodag/plugins/base.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Dict, List, Tuple
+from typing import TYPE_CHECKING, Any
 
 from eodag.utils.exceptions import PluginNotFoundError
 
@@ -29,21 +29,21 @@ class EODAGPluginMount(type):
     """Plugin mount"""
 
     def __init__(
-        cls, name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]
+        cls, name: str, bases: tuple[type, ...], attrs: dict[str, Any]
     ) -> None:
         if not hasattr(cls, "plugins"):
             # This branch only executes when processing the mount point itself.
             # So, since this is a new plugin type, not an implementation, this
             # class shouldn't be registered as a plugin. Instead, it sets up a
             # list where plugins can be registered later.
-            cls.plugins: List[EODAGPluginMount] = []
+            cls.plugins: list[EODAGPluginMount] = []
         else:
             # This must be a plugin implementation, which should be registered.
             # Simply appending it to the list is all that's needed to keep
             # track of it later.
             cls.plugins.append(cls)
 
-    def get_plugins(cls, *args: Any, **kwargs: Any) -> List[EODAGPluginMount]:
+    def get_plugins(cls, *args: Any, **kwargs: Any) -> list[EODAGPluginMount]:
         """Get plugins"""
         return [plugin(*args, **kwargs) for plugin in cls.plugins]
 

eodag/plugins/crunch/base.py

@@ -17,7 +17,7 @@
 # limitations under the License
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from eodag.config import PluginConfig
 from eodag.plugins.base import PluginTopic
@@ -27,14 +27,17 @@ if TYPE_CHECKING:
 
 
 class Crunch(PluginTopic):
-    """Base cruncher"""
+    """Base cruncher
 
-    def __init__(self, config: Optional[Dict[str, Any]]) -> None:
+    :param config: Crunch configuration
+    """
+
+    def __init__(self, config: Optional[dict[str, Any]]) -> None:
         self.config = PluginConfig()
         self.config.__dict__ = config if config is not None else {}
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: Any
+    ) -> list[EOProduct]:
         """Implementation of how the results must be crunched"""
         raise NotImplementedError

eodag/plugins/crunch/filter_date.py

@@ -21,7 +21,7 @@ import datetime
 import logging
 import time
 from datetime import datetime as dt
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
 
 import dateutil.parser
 from dateutil import tz
@@ -37,10 +37,13 @@ logger = logging.getLogger("eodag.crunch.date")
 class FilterDate(Crunch):
     """FilterDate cruncher: filter products by date
 
+    Allows to filter out products that are older than a start date (optional) or more recent than an end date
+    (optional).
+
     :param config: Crunch configuration, may contain :
 
-                   - `start`: (optional) start sensing time in iso format
-                   - `end`: (optional) end sensing time in iso format
+        * ``start`` (``str``): start sensing time in iso format
+        * ``end`` (``str``): end sensing time in iso format
     """
 
     @staticmethod
@@ -54,8 +57,8 @@ class FilterDate(Crunch):
         return dateutil.parser.parse(start_date)
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: Any
+    ) -> list[EOProduct]:
         """Execute crunch: Filter products between start and end dates.
 
         :param products: A list of products resulting from a search
@@ -86,7 +89,7 @@ class FilterDate(Crunch):
         if not filter_start and not filter_end:
             return products
 
-        filtered: List[EOProduct] = []
+        filtered: list[EOProduct] = []
         for product in products:
 
             # product start date

eodag/plugins/crunch/filter_latest_intersect.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import datetime
 import logging
 import time
-from typing import TYPE_CHECKING, Any, Dict, List, Union
+from typing import TYPE_CHECKING, Any, Union
 
 import dateutil.parser
 from shapely import geometry
@@ -39,7 +39,8 @@ logger = logging.getLogger("eodag.crunch.latest_intersect")
 class FilterLatestIntersect(Crunch):
     """FilterLatestIntersect cruncher
 
-    Filter latest products (the ones with a the highest start date) that intersect search extent
+    Filter latest products (the ones with a the highest start date) that intersect search extent;
+    The configuration for this plugin is an empty dict
     """
 
     @staticmethod
@@ -53,14 +54,14 @@ class FilterLatestIntersect(Crunch):
         return dateutil.parser.parse(start_date)
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: dict[str, Any]
+    ) -> list[EOProduct]:
         """Execute crunch:
         Filter latest products (the ones with a the highest start date) that intersect search extent.
 
         :param products: A list of products resulting from a search
-        :param search_params: Search criteria that must contain `geometry` (dict)
-                              or search `geom` (:class:`shapely.geometry.base.BaseGeometry`) argument will be used
+        :param search_params: Search criteria that must contain ``geometry`` or ``geom`` parameters having value of
+                              type :class:`shapely.geometry.base.BaseGeometry` or ``dict[str, Any]``
         :returns: The filtered products
         """
         logger.debug("Start filtering for latest products")
@@ -68,9 +69,9 @@ class FilterLatestIntersect(Crunch):
             return []
         # Warning: May crash if startTimeFromAscendingNode is not in the appropriate format
         products.sort(key=self.sort_product_by_start_date, reverse=True)
-        filtered: List[EOProduct] = []
+        filtered: list[EOProduct] = []
         add_to_filtered = filtered.append
-        footprint: Union[Dict[str, Any], BaseGeometry, Any] = search_params.get(
+        footprint: Union[dict[str, Any], BaseGeometry, Any] = search_params.get(
             "geometry"
         ) or search_params.get("geom")
         if not footprint:

eodag/plugins/crunch/filter_latest_tpl_name.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Match, Optional, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 from eodag.plugins.crunch.base import Crunch
 from eodag.utils.exceptions import ValidationError
@@ -37,12 +37,12 @@ class FilterLatestByName(Crunch):
 
     :param config: Crunch configuration, must contain :
 
-                   - `name_pattern` : product name pattern
+        * ``name_pattern`` (``str``) (**mandatory**): product name pattern
     """
 
     NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P<tileid>\\d\{6\}\)")
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: dict[str, Any]) -> None:
         super(FilterLatestByName, self).__init__(config)
         name_pattern = config.pop("name_pattern")
         if not self.NAME_PATTERN_CONSTRAINT.search(name_pattern):
@@ -54,19 +54,19 @@ class FilterLatestByName(Crunch):
         self.name_pattern = re.compile(name_pattern)
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: Any
+    ) -> list[EOProduct]:
         """Execute crunch: Filter Search results to get only the latest product, based on the name of the product
 
         :param products: A list of products resulting from a search
         :returns: The filtered products
         """
         logger.debug("Starting products filtering")
-        processed: List[str] = []
-        filtered: List[EOProduct] = []
+        processed: list[str] = []
+        filtered: list[EOProduct] = []
         for product in products:
             match = cast(
-                Optional[Match[Any]],
+                Optional[re.Match[Any]],
                 self.name_pattern.match(product.properties["title"]),
             )
             if match:

eodag/plugins/crunch/filter_overlap.py

@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
 
 from eodag.plugins.crunch.base import Crunch
 from eodag.utils import get_geometry_from_various
@@ -40,19 +40,17 @@ class FilterOverlap(Crunch):
 
     Filter products, retaining only those that are overlapping with the search_extent
 
-    :param config: Crunch configuration, may contain :
+    :param config: Crunch configuration may contain the following parameters which are mutually exclusive:
 
-                   - `minimum_overlap` : minimal overlap percentage
-                   - `contains` : True if product geometry contains the search area
-                   - `intersects` : True if product geometry intersects the search area
-                   - `within` : True if product geometry is within the search area
-
-                   These configuration parameters are mutually exclusive.
+        * ``minimum_overlap`` (``Union[float, str]``): minimal overlap percentage; default: ``"0"``
+        * ``contains`` (``bool``): ``True`` if product geometry contains the search area; default: ``False``
+        * ``intersects`` (``bool``): ``True`` if product geometry intersects the search area; default: ``False``
+        * ``within`` (``bool``): ``True`` if product geometry is within the search area; default: ``False``
     """
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: Any
+    ) -> list[EOProduct]:
         """Execute crunch: Filter products, retaining only those that are overlapping with the search_extent
 
         :param products: A list of products resulting from a search
@@ -60,7 +58,7 @@ class FilterOverlap(Crunch):
         :returns: The filtered products
         """
         logger.debug("Start filtering for overlapping products")
-        filtered: List[EOProduct] = []
+        filtered: list[EOProduct] = []
         add_to_filtered = filtered.append
 
         search_geom = get_geometry_from_various(**search_params)

eodag/plugins/crunch/filter_property.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 import operator
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
 
 from eodag.plugins.crunch.base import Crunch
 
@@ -34,15 +34,16 @@ class FilterProperty(Crunch):
 
     Filter products, retaining only those whose property match criteria
 
-    :param config: Crunch configuration, should contain :
+    :param config: Crunch configuration, must contain :
 
-                   - `property=value` : property key from product.properties, associated to its filter value
-                   - `operator` : (optional) Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq`
+        * ``<property>`` ``(Any)`` (**mandatory**): property key from ``product.properties``, associated to its filter
+          value
+        * ``operator`` (``str``): Operator used for filtering (one of ``lt,le,eq,ne,ge,gt``). Default is ``eq``
     """
 
     def proceed(
-        self, products: List[EOProduct], **search_params: Any
-    ) -> List[EOProduct]:
+        self, products: list[EOProduct], **search_params: Any
+    ) -> list[EOProduct]:
         """Execute crunch: Filter products, retaining only those that match property filtering
 
         :param products: A list of products resulting from a search
@@ -71,16 +72,15 @@ class FilterProperty(Crunch):
             property_key,
             property_value,
         )
-        filtered: List[EOProduct] = []
+        filtered: list[EOProduct] = []
         add_to_filtered = filtered.append
 
         for product in products:
             if property_key not in product.properties.keys():
                 logger.warning(
-                    "%s not found in product.properties, filtering disabled.",
-                    property_key,
+                    f"{property_key} not found in {product}.properties, product skipped",
                 )
-                return products
+                continue
             if operator_method(product.properties[property_key], property_value):
                 add_to_filtered(product)