eodag 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

eodag/api/core.py

@@ -30,10 +30,6 @@ from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
 
 import geojson
 import yaml.parser
-from whoosh import analysis, fields
-from whoosh.fields import Schema
-from whoosh.index import exists_in, open_dir
-from whoosh.qparser import QueryParser
 
 from eodag.api.product.metadata_mapping import (
     NOT_AVAILABLE,
@@ -61,7 +57,6 @@ from eodag.plugins.search.build_search_result import MeteoblueSearch
 from eodag.plugins.search.qssearch import PostJsonSearch
 from eodag.types import model_fields_to_annotated
 from eodag.types.queryables import CommonQueryables, QueryablesDict
-from eodag.types.whoosh import EODAGQueryParser, create_in
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
@@ -75,7 +70,6 @@ from eodag.utils import (
     _deprecated,
     get_geometry_from_various,
     makedirs,
-    obj_md5sum,
     sort_dict,
     string_to_jsonpath,
     uri_to_path,
@@ -83,19 +77,18 @@ from eodag.utils import (
 from eodag.utils.env import is_env_var_true
 from eodag.utils.exceptions import (
     AuthenticationError,
-    EodagError,
     NoMatchingProductType,
     PluginImplementationError,
     RequestError,
     UnsupportedProductType,
     UnsupportedProvider,
 )
+from eodag.utils.free_text_search import compile_free_text_query
 from eodag.utils.rest import rfc3339_str_to_datetime
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:
     from shapely.geometry.base import BaseGeometry
-    from whoosh.index import Index
 
     from eodag.api.product import EOProduct
     from eodag.plugins.apis.base import Api
@@ -125,7 +118,6 @@ class EODataAccessGateway:
             res_files("eodag") / "resources" / "product_types.yml"
         )
         self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
-        self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
         self.providers_config = load_default_config()
 
         env_var_cfg_dir = "EODAG_CFG_DIR"
@@ -189,6 +181,8 @@ class EODataAccessGateway:
             self._sync_provider_product_types(
                 provider, available_product_types, strict_mode
             )
+        # init product types configuration
+        self._product_types_config_init()
 
         # re-build _plugins_manager using up-to-date providers_config
         self._plugins_manager.rebuild(self.providers_config)
@@ -201,10 +195,6 @@ class EODataAccessGateway:
         # Sort providers taking into account of possible new priority orders
         self._plugins_manager.sort_providers()
 
-        # Build a search index for product types
-        self._product_types_index: Optional[Index] = None
-        self.build_index()
-
         # set locations configuration
         if locations_conf_path is None:
             locations_conf_path = os.getenv("EODAG_LOCS_CFG_FILE")
@@ -235,6 +225,11 @@ class EODataAccessGateway:
                     )
         self.set_locations_conf(locations_conf_path)
 
+    def _product_types_config_init(self) -> None:
+        """Initialize product types configuration."""
+        for pt_id, pd_dict in self.product_types_config.source.items():
+            self.product_types_config.source[pt_id].setdefault("_id", pt_id)
+
     def _sync_provider_product_types(
         self,
         provider: str,
@@ -294,95 +289,6 @@ class EODataAccessGateway:
         """Get eodag package version"""
         return version("eodag")
 
-    def build_index(self) -> None:
-        """Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
-        index for product types searches.
-        """
-        index_dir = os.path.join(self.conf_dir, ".index")
-
-        try:
-            create_index = not exists_in(index_dir)
-        except ValueError as ve:
-            # Whoosh uses pickle internally. New versions of Python sometimes introduce
-            # a new pickle protocol (e.g. 3.4 -> 4, 3.8 -> 5), the new version not
-            # being supported by previous versions of Python (e.g. Python 3.7 doesn't
-            # support Protocol 5). In that case, we need to recreate the .index.
-            if "unsupported pickle protocol" in str(ve):
-                logger.debug("Need to recreate whoosh .index: '%s'", ve)
-                create_index = True
-            # Unexpected error
-            else:
-                logger.error(
-                    "Error while opening .index using whoosh, "
-                    "please report this issue and try to delete '%s' manually",
-                    index_dir,
-                )
-                raise
-        # check index version
-        if not create_index:
-            if self._product_types_index is None:
-                logger.debug("Opening product types index in %s", index_dir)
-                self._product_types_index = open_dir(index_dir)
-
-            with self._product_types_index.searcher() as searcher:
-                p = QueryParser("md5", self._product_types_index.schema, plugins=[])
-                query = p.parse(self.product_types_config_md5)
-                results = searcher.search(query, limit=1)
-
-                if not results:
-                    create_index = True
-                    logger.debug(
-                        "Out-of-date product types index removed from %s", index_dir
-                    )
-
-        if create_index:
-            logger.debug("Creating product types index in %s", index_dir)
-            makedirs(index_dir)
-
-            kw_analyzer = (
-                analysis.CommaSeparatedTokenizer()
-                | analysis.LowercaseFilter()
-                | analysis.SubstitutionFilter("-", "")
-                | analysis.SubstitutionFilter("_", "")
-            )
-
-            product_types_schema = Schema(
-                ID=fields.ID(stored=True),
-                abstract=fields.TEXT,
-                instrument=fields.IDLIST,
-                platform=fields.ID,
-                platformSerialIdentifier=fields.IDLIST,
-                processingLevel=fields.ID,
-                sensorType=fields.ID,
-                md5=fields.ID,
-                license=fields.ID,
-                title=fields.TEXT,
-                missionStartDate=fields.STORED,
-                missionEndDate=fields.STORED,
-                keywords=fields.KEYWORD(analyzer=kw_analyzer),
-                stacCollection=fields.STORED,
-            )
-            self._product_types_index = create_in(index_dir, product_types_schema)
-            ix_writer = self._product_types_index.writer()
-            for product_type in self.list_product_types(fetch_providers=False):
-                versioned_product_type = dict(
-                    product_type, **{"md5": self.product_types_config_md5}
-                )
-                # add to index
-                try:
-                    ix_writer.add_document(
-                        **{
-                            k: v
-                            for k, v in versioned_product_type.items()
-                            if k in product_types_schema.names()
-                        }
-                    )
-                except TypeError as e:
-                    logger.error(
-                        f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
-                    )
-            ix_writer.commit()
-
     def set_preferred_provider(self, provider: str) -> None:
         """Set max priority for the given provider.
 
@@ -674,8 +580,6 @@ class EODataAccessGateway:
                     continue
 
                 config = self.product_types_config[product_type_id]
-                config["_id"] = product_type_id
-
                 if "alias" in config:
                     product_type_id = config["alias"]
                 product_type = {"ID": product_type_id, **config}
@@ -977,14 +881,12 @@ class EODataAccessGateway:
                             # to self.product_types_config
                             self.product_types_config.source.update(
                                 {
-                                    new_product_type: new_product_types_conf[
-                                        "product_types_config"
-                                    ][new_product_type]
+                                    new_product_type: {"_id": new_product_type}
+                                    | new_product_types_conf["product_types_config"][
+                                        new_product_type
+                                    ]
                                 }
                             )
-                            self.product_types_config_md5 = obj_md5sum(
-                                self.product_types_config.source
-                            )
                             ext_product_types_conf[provider] = new_product_types_conf
                             new_product_types.append(new_product_type)
                 if new_product_types:
@@ -1000,9 +902,6 @@ class EODataAccessGateway:
         # re-create _plugins_manager using up-to-date providers_config
         self._plugins_manager.build_product_type_to_provider_config_map()
 
-        # rebuild index after product types list update
-        self.build_index()
-
     def available_providers(
         self, product_type: Optional[str] = None, by_group: bool = False
     ) -> list[str]:
@@ -1103,11 +1002,11 @@ class EODataAccessGateway:
         """
         Find EODAG product type IDs that best match a set of search parameters.
 
-        See https://whoosh.readthedocs.io/en/latest/querylang.html#the-default-query-language
-          for syntax.
+        When using several filters, product types that match most of them will be returned at first.
 
-        :param free_text: Whoosh-compatible free text search filter used to search
-                        accross all the following parameters
+        :param free_text: Free text search filter used to search accross all the following parameters. Handles logical
+                          operators with parenthesis (``AND``/``OR``/``NOT``), quoted phrases (``"exact phrase"``),
+                          ``*`` and ``?`` wildcards.
         :param intersect: Join results for each parameter using INTERSECT instead of UNION.
         :param instrument: Instrument parameter.
         :param platform: Platform parameter.
@@ -1125,69 +1024,105 @@ class EODataAccessGateway:
         if productType := kwargs.get("productType"):
             return [productType]
 
-        if not self._product_types_index:
-            raise EodagError("Missing product types index")
-
-        filters = {
-            "instrument": instrument,
-            "platform": platform,
-            "platformSerialIdentifier": platformSerialIdentifier,
-            "processingLevel": processingLevel,
-            "sensorType": sensorType,
-            "keywords": keywords,
-            "abstract": abstract,
-            "title": title,
+        filters: dict[str, str] = {
+            k: v
+            for k, v in {
+                "instrument": instrument,
+                "platform": platform,
+                "platformSerialIdentifier": platformSerialIdentifier,
+                "processingLevel": processingLevel,
+                "sensorType": sensorType,
+                "keywords": keywords,
+                "abstract": abstract,
+                "title": title,
+            }.items()
+            if v is not None
         }
-        joint = " AND " if intersect else " OR "
-        filters_text = joint.join(
-            [f"{k}:({v})" for k, v in filters.items() if v is not None]
+
+        only_dates = (
+            True
+            if (not free_text and not filters and (missionStartDate or missionEndDate))
+            else False
         )
 
-        text = f"({free_text})" if free_text else ""
-        if free_text and filters_text:
-            text += joint
-        if filters_text:
-            text += f"({filters_text})"
-
-        if not text and (missionStartDate or missionEndDate):
-            text = "*"
-
-        with self._product_types_index.searcher() as searcher:
-            p = EODAGQueryParser(list(filters.keys()), self._product_types_index.schema)
-            query = p.parse(text)
-            results = searcher.search(query, limit=None)
-
-            guesses: list[dict[str, str]] = [dict(r) for r in results or []]
-
-        # datetime filtering
-        if missionStartDate or missionEndDate:
-            min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
-            max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
-            guesses = [
-                g
-                for g in guesses
-                if (
-                    max(
-                        rfc3339_str_to_datetime(missionStartDate)
-                        if missionStartDate
-                        else min_aware,
-                        rfc3339_str_to_datetime(g["missionStartDate"])
-                        if g.get("missionStartDate")
-                        else min_aware,
-                    )
-                    <= min(
-                        rfc3339_str_to_datetime(missionEndDate)
-                        if missionEndDate
-                        else max_aware,
-                        rfc3339_str_to_datetime(g["missionEndDate"])
-                        if g.get("missionEndDate")
-                        else max_aware,
-                    )
+        free_text_evaluator = (
+            compile_free_text_query(free_text) if free_text else lambda _: True
+        )
+
+        guesses_with_score: list[tuple[str, int]] = []
+
+        for pt_id, pt_dict in self.product_types_config.source.items():
+            if (
+                pt_id == GENERIC_PRODUCT_TYPE
+                or pt_id
+                not in self._plugins_manager.product_type_to_provider_config_map
+            ):
+                continue
+
+            score = 0  # how many filters matched
+
+            # free text search
+            if free_text:
+                match = free_text_evaluator(pt_dict)
+                if match:
+                    score += 1
+                elif intersect:
+                    continue  # must match all filters
+
+            # individual filters
+            if filters:
+                filters_matching_method = all if intersect else any
+                filters_evaluators = {
+                    filter_name: compile_free_text_query(value)
+                    for filter_name, value in filters.items()
+                    if value is not None
+                }
+
+                filter_matches = [
+                    filters_evaluators[filter_name]({filter_name: pt_dict[filter_name]})
+                    for filter_name, value in filters.items()
+                    if filter_name in pt_dict
+                ]
+
+                if filters_matching_method(filter_matches):
+                    # add number of True matches to score
+                    score += sum(filter_matches)
+                elif intersect:
+                    continue  # must match all filters
+
+            if score == 0 and not only_dates:
+                continue
+
+            # datetime filtering
+            if missionStartDate or missionEndDate:
+                min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
+                max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
+
+                max_start = max(
+                    rfc3339_str_to_datetime(missionStartDate)
+                    if missionStartDate
+                    else min_aware,
+                    rfc3339_str_to_datetime(pt_dict["missionStartDate"])
+                    if pt_dict.get("missionStartDate")
+                    else min_aware,
                 )
-            ]
+                min_end = min(
+                    rfc3339_str_to_datetime(missionEndDate)
+                    if missionEndDate
+                    else max_aware,
+                    rfc3339_str_to_datetime(pt_dict["missionEndDate"])
+                    if pt_dict.get("missionEndDate")
+                    else max_aware,
+                )
+                if not (max_start <= min_end):
+                    continue
 
-        if guesses:
-            return [g["ID"] for g in guesses or []]
+            guesses_with_score.append((pt_id, score))
+
+        if guesses_with_score:
+            # sort by score descending, then pt_id for stability
+            guesses_with_score.sort(key=lambda x: (-x[1], x[0]))
+            return [pt_id for pt_id, _ in guesses_with_score]
 
         raise NoMatchingProductType()
 
@@ -2007,20 +1942,6 @@ class EODataAccessGateway:
                     nb_res,
                     search_plugin.provider,
                 )
-                # Hitting for instance
-                # https://theia.cnes.fr/atdistrib/resto2/api/collections/SENTINEL2/
-                #   search.json?startDate=2019-03-01&completionDate=2019-06-15
-                #   &processingLevel=LEVEL2A&maxRecords=1&page=1
-                # returns a number (properties.totalResults) that is the number of
-                # products in the collection (here SENTINEL2) instead of the estimated
-                # total number of products matching the search criteria (start/end date).
-                # Remove this warning when this is fixed upstream by THEIA.
-                if search_plugin.provider == "theia":
-                    logger.warning(
-                        "Results found on provider 'theia' is the total number of products "
-                        "available in the searched collection (e.g. SENTINEL2) instead of "
-                        "the total number of products matching the search criteria"
-                    )
         except Exception as e:
             if raise_errors:
                 # Raise the error, letting the application wrapping eodag know that

eodag/api/product/metadata_mapping.py

@@ -42,6 +42,7 @@ from shapely.ops import transform
 from eodag.types.queryables import Queryables
 from eodag.utils import (
     DEFAULT_PROJ,
+    _deprecated,
     deepcopy,
     dict_items_recursive_apply,
     format_string,
@@ -180,6 +181,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``to_datetime_dict``: convert a datetime string to a dictionary where values are either a string or a list
         - ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
         - ``sanitize``: sanitize string
+        - ``ceda_collection_name``: generate a CEDA collection name from a string
 
     :param search_param: The string to be formatted
     :param args: (optional) Additional arguments to use in the formatting process
@@ -219,7 +221,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
                 elif value is not None:
                     converted = self.custom_converter(value)
                 else:
-                    converted = ""
+                    converted = None
                 # Clear this state variable in case the same converter is used to
                 # resolve other named arguments
                 self.custom_converter = None
@@ -374,6 +376,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         def convert_to_geojson(value: Any) -> str:
             return geojson.dumps(value)
 
+        @staticmethod
+        def convert_to_geojson_polytope(
+            value: BaseGeometry,
+        ) -> Union[dict[Any, Any], str]:
+            # ECMWF Polytope uses non-geojson structure for features
+            if isinstance(value, Polygon):
+                return {
+                    "type": "polygon",
+                    "shape": [[y, x] for x, y in value.exterior.coords],
+                }
+            raise ValidationError("to_geojson_polytope only accepts shapely Polygon")
+
         @staticmethod
         def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
             """Convert EWKT (Extended Well-Known text) to shapely geometry"""
@@ -488,10 +502,14 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
 
         @staticmethod
         def convert_get_group_name(string: str, pattern: str) -> str:
+            sanitized_pattern = pattern.replace(" ", "_SPACE_")
             try:
-                match = re.search(pattern, str(string))
+                match = re.search(sanitized_pattern, str(string))
                 if match:
-                    return match.lastgroup or NOT_AVAILABLE
+                    if result := match.lastgroup:
+                        return result.replace("_SPACE_", " ")
+                    else:
+                        return NOT_AVAILABLE
             except AttributeError:
                 pass
             logger.warning(
@@ -511,6 +529,14 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             old, new = ast.literal_eval(args)
             return re.sub(old, new, value)
 
+        @staticmethod
+        def convert_ceda_collection_name(value: str) -> str:
+            data_regex = re.compile(r"/data/(?P<name>.+?)/?$")
+            match = data_regex.search(value)
+            if match:
+                return match.group("name").replace("/", "_").upper()
+            return "NOT_AVAILABLE"
+
         @staticmethod
         def convert_recursive_sub_str(
             input_obj: Union[dict[Any, Any], list[Any]], args: str
@@ -615,6 +641,10 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
                 return NOT_AVAILABLE
 
         @staticmethod
+        @_deprecated(
+            reason="Method that was used in previous wekeo provider configuration, but not used anymore",
+            version="3.7.1",
+        )
         def convert_split_id_into_s1_params(product_id: str) -> dict[str, str]:
             parts: list[str] = re.split(r"_(?!_)", product_id)
             if len(parts) < 9:
@@ -667,6 +697,10 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             return params
 
         @staticmethod
+        @_deprecated(
+            reason="Method that was used in previous wekeo provider configuration, but not used anymore",
+            version="3.7.1",
+        )
         def convert_split_id_into_s5p_params(product_id: str) -> dict[str, str]:
             parts: list[str] = re.split(r"_(?!_)", product_id)
             params = {
@@ -685,6 +719,10 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             return params
 
         @staticmethod
+        @_deprecated(
+            reason="Method that was used in previous wekeo provider configuration, but not used anymore",
+            version="3.7.1",
+        )
         def convert_split_cop_dem_id(product_id: str) -> list[int]:
             parts = product_id.split("_")
             lattitude = parts[3]
@@ -1342,6 +1380,7 @@ def format_query_params(
                     formatted_query_param = remove_str_array_quotes(
                         formatted_query_param
                     )
+
                     # json query string (for POST request)
                     update_nested_dict(
                         query_params,

eodag/cli.py

@@ -49,11 +49,12 @@ import sys
 import textwrap
 from importlib.metadata import metadata
 from typing import TYPE_CHECKING, Any, Mapping
+from urllib.parse import parse_qs
 
 import click
 
 from eodag.api.core import EODataAccessGateway, SearchResult
-from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
+from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
 from eodag.utils.logging import setup_logging
 
@@ -109,8 +110,9 @@ class MutuallyExclusiveOption(click.Option):
         """Raise error or use parent handle_parse_result()"""
         if self.mutually_exclusive.intersection(opts) and self.name in opts:
             raise click.UsageError(
-                "Illegal usage: `{}` is mutually exclusive with "
-                "arguments `{}`.".format(self.name, ", ".join(self.mutually_exclusive))
+                "Illegal usage: `{}` is mutually exclusive with arguments `{}`.".format(
+                    self.name, ", ".join(self.mutually_exclusive)
+                )
             )
 
         return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)
@@ -687,6 +689,7 @@ def serve_rest(
     setup_logging(verbose=ctx.obj["verbosity"])
     try:
         import uvicorn
+        import uvicorn.config
     except ImportError:
         raise ImportError(
             "Feature not available, please install eodag[server] or eodag[all]"

eodag/config.py

@@ -307,7 +307,8 @@ class PluginConfig(yaml.YAMLObject):
         single_collection_fetch_url: str
         #: Query string to be added to the fetch_url to filter for a collection
         single_collection_fetch_qs: str
-        #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
+        #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url. If ``ID``
+        #: is redefined in this mapping, it will replace ``generic_product_type_id`` value
         single_product_type_parsable_metadata: dict[str, str]
 
     class DiscoverQueryables(TypedDict, total=False):
@@ -452,6 +453,11 @@ class PluginConfig(yaml.YAMLObject):
     discover_queryables: PluginConfig.DiscoverQueryables
     #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
     metadata_mapping: dict[str, Union[str, list[str]]]
+    #: :class:`~eodag.plugins.search.base.Search` :attr:`~eodag.config.PluginConfig.metadata_mapping` got from the given
+    #: product type
+    metadata_mapping_from_product: str
+    #: :class:`~eodag.plugins.search.base.Search` A mapping for the metadata of individual assets
+    assets_mapping: dict[str, dict[str, Any]]
     #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
     remove_from_queryables: list[str]
     #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string

eodag/plugins/authentication/openid_connect.py

@@ -23,6 +23,7 @@ import string
 from datetime import datetime, timedelta, timezone
 from random import SystemRandom
 from typing import TYPE_CHECKING, Any, Optional
+from urllib.parse import parse_qs, urlparse
 
 import jwt
 import requests
@@ -34,9 +35,7 @@ from eodag.utils import (
     DEFAULT_TOKEN_EXPIRATION_MARGIN,
     HTTP_REQ_TIMEOUT,
     USER_AGENT,
-    parse_qs,
     repeatfunc,
-    urlparse,
 )
 from eodag.utils.exceptions import (
     AuthenticationError,