eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

eodag/types/queryables.py

@@ -15,11 +15,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Annotated, Optional
+from __future__ import annotations
+
+from collections import UserDict
+from datetime import date, datetime
+from typing import Annotated, Any, Optional, Union
 
 from annotated_types import Lt
 from pydantic import BaseModel, Field
+from pydantic.fields import FieldInfo
 from pydantic.types import PositiveInt
+from pydantic_core import PydanticUndefined
+
+from eodag.types import annotated_dict_to_model, model_fields_to_annotated
+from eodag.utils.repr import remove_class_repr, shorter_type_repr
 
 Percentage = Annotated[PositiveInt, Lt(100)]
 
@@ -28,17 +37,11 @@ class CommonQueryables(BaseModel):
     """A class representing search common queryable properties."""
 
     productType: Annotated[str, Field()]
-    id: Annotated[Optional[str], Field(None)]
-    start: Annotated[Optional[str], Field(None, alias="startTimeFromAscendingNode")]
-    end: Annotated[Optional[str], Field(None, alias="completionTimeFromAscendingNode")]
-    geom: Annotated[Optional[str], Field(None, alias="geometry")]
 
     @classmethod
     def get_queryable_from_alias(cls, value: str) -> str:
         """Get queryable parameter from alias
 
-        >>> CommonQueryables.get_queryable_from_alias('startTimeFromAscendingNode')
-        'start'
         >>> CommonQueryables.get_queryable_from_alias('productType')
         'productType'
         """
@@ -49,75 +52,183 @@ class CommonQueryables(BaseModel):
         }
         return alias_map.get(value, value)
 
+    @classmethod
+    def get_with_default(
+        cls, field: str, default: Optional[Any]
+    ) -> Annotated[Any, FieldInfo]:
+        """Get field and set default value."""
+        annotated_fields = model_fields_to_annotated(cls.model_fields)
+        f = annotated_fields[field]
+        if default is None:
+            return f
+        f.__metadata__[0].default = default
+        return f
+
 
 class Queryables(CommonQueryables):
-    """A class representing all search queryable properties."""
+    """A class representing all search queryable properties.
 
-    uid: Annotated[Optional[str], Field(None)]
+    Parameters default value is set to ``None`` to have them not required.
+    """
+
+    start: Annotated[
+        Union[datetime, date], Field(None, alias="startTimeFromAscendingNode")
+    ]
+    end: Annotated[
+        Union[datetime, date], Field(None, alias="completionTimeFromAscendingNode")
+    ]
+    geom: Annotated[str, Field(None, alias="geometry")]
+    uid: Annotated[str, Field(None)]
     # OpenSearch Parameters for Collection Search (Table 3)
-    doi: Annotated[Optional[str], Field(None)]
-    platform: Annotated[Optional[str], Field(None)]
-    platformSerialIdentifier: Annotated[Optional[str], Field(None)]
-    instrument: Annotated[Optional[str], Field(None)]
-    sensorType: Annotated[Optional[str], Field(None)]
-    compositeType: Annotated[Optional[str], Field(None)]
-    processingLevel: Annotated[Optional[str], Field(None)]
-    orbitType: Annotated[Optional[str], Field(None)]
-    spectralRange: Annotated[Optional[str], Field(None)]
-    wavelengths: Annotated[Optional[str], Field(None)]
-    hasSecurityConstraints: Annotated[Optional[str], Field(None)]
-    dissemination: Annotated[Optional[str], Field(None)]
+    doi: Annotated[str, Field(None)]
+    platform: Annotated[str, Field(None)]
+    platformSerialIdentifier: Annotated[str, Field(None)]
+    instrument: Annotated[str, Field(None)]
+    sensorType: Annotated[str, Field(None)]
+    compositeType: Annotated[str, Field(None)]
+    processingLevel: Annotated[str, Field(None)]
+    orbitType: Annotated[str, Field(None)]
+    spectralRange: Annotated[str, Field(None)]
+    wavelengths: Annotated[str, Field(None)]
+    hasSecurityConstraints: Annotated[str, Field(None)]
+    dissemination: Annotated[str, Field(None)]
     # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
-    title: Annotated[Optional[str], Field(None)]
-    topicCategory: Annotated[Optional[str], Field(None)]
-    keyword: Annotated[Optional[str], Field(None)]
-    abstract: Annotated[Optional[str], Field(None)]
-    resolution: Annotated[Optional[int], Field(None)]
-    organisationName: Annotated[Optional[str], Field(None)]
-    organisationRole: Annotated[Optional[str], Field(None)]
-    publicationDate: Annotated[Optional[str], Field(None)]
-    lineage: Annotated[Optional[str], Field(None)]
-    useLimitation: Annotated[Optional[str], Field(None)]
-    accessConstraint: Annotated[Optional[str], Field(None)]
-    otherConstraint: Annotated[Optional[str], Field(None)]
-    classification: Annotated[Optional[str], Field(None)]
-    language: Annotated[Optional[str], Field(None)]
-    specification: Annotated[Optional[str], Field(None)]
+    title: Annotated[str, Field(None)]
+    topicCategory: Annotated[str, Field(None)]
+    keyword: Annotated[str, Field(None)]
+    abstract: Annotated[str, Field(None)]
+    resolution: Annotated[int, Field(None)]
+    organisationName: Annotated[str, Field(None)]
+    organisationRole: Annotated[str, Field(None)]
+    publicationDate: Annotated[str, Field(None)]
+    lineage: Annotated[str, Field(None)]
+    useLimitation: Annotated[str, Field(None)]
+    accessConstraint: Annotated[str, Field(None)]
+    otherConstraint: Annotated[str, Field(None)]
+    classification: Annotated[str, Field(None)]
+    language: Annotated[str, Field(None)]
+    specification: Annotated[str, Field(None)]
     # OpenSearch Parameters for Product Search (Table 5)
-    parentIdentifier: Annotated[Optional[str], Field(None)]
-    productionStatus: Annotated[Optional[str], Field(None)]
-    acquisitionType: Annotated[Optional[str], Field(None)]
-    orbitNumber: Annotated[Optional[int], Field(None)]
-    orbitDirection: Annotated[Optional[str], Field(None)]
-    track: Annotated[Optional[str], Field(None)]
-    frame: Annotated[Optional[str], Field(None)]
-    swathIdentifier: Annotated[Optional[str], Field(None)]
-    cloudCover: Annotated[Optional[Percentage], Field(None)]
-    snowCover: Annotated[Optional[Percentage], Field(None)]
-    lowestLocation: Annotated[Optional[str], Field(None)]
-    highestLocation: Annotated[Optional[str], Field(None)]
-    productVersion: Annotated[Optional[str], Field(None)]
-    productQualityStatus: Annotated[Optional[str], Field(None)]
-    productQualityDegradationTag: Annotated[Optional[str], Field(None)]
-    processorName: Annotated[Optional[str], Field(None)]
-    processingCenter: Annotated[Optional[str], Field(None)]
-    creationDate: Annotated[Optional[str], Field(None)]
-    modificationDate: Annotated[Optional[str], Field(None)]
-    processingDate: Annotated[Optional[str], Field(None)]
-    sensorMode: Annotated[Optional[str], Field(None)]
-    archivingCenter: Annotated[Optional[str], Field(None)]
-    processingMode: Annotated[Optional[str], Field(None)]
+    parentIdentifier: Annotated[str, Field(None)]
+    productionStatus: Annotated[str, Field(None)]
+    acquisitionType: Annotated[str, Field(None)]
+    orbitNumber: Annotated[int, Field(None)]
+    orbitDirection: Annotated[str, Field(None)]
+    track: Annotated[str, Field(None)]
+    frame: Annotated[str, Field(None)]
+    swathIdentifier: Annotated[str, Field(None)]
+    cloudCover: Annotated[Percentage, Field(None)]
+    snowCover: Annotated[Percentage, Field(None)]
+    lowestLocation: Annotated[str, Field(None)]
+    highestLocation: Annotated[str, Field(None)]
+    productVersion: Annotated[str, Field(None)]
+    productQualityStatus: Annotated[str, Field(None)]
+    productQualityDegradationTag: Annotated[str, Field(None)]
+    processorName: Annotated[str, Field(None)]
+    processingCenter: Annotated[str, Field(None)]
+    creationDate: Annotated[str, Field(None)]
+    modificationDate: Annotated[str, Field(None)]
+    processingDate: Annotated[str, Field(None)]
+    sensorMode: Annotated[str, Field(None)]
+    archivingCenter: Annotated[str, Field(None)]
+    processingMode: Annotated[str, Field(None)]
     # OpenSearch Parameters for Acquistion Parameters Search (Table 6)
-    availabilityTime: Annotated[Optional[str], Field(None)]
-    acquisitionStation: Annotated[Optional[str], Field(None)]
-    acquisitionSubType: Annotated[Optional[str], Field(None)]
-    illuminationAzimuthAngle: Annotated[Optional[str], Field(None)]
-    illuminationZenithAngle: Annotated[Optional[str], Field(None)]
-    illuminationElevationAngle: Annotated[Optional[str], Field(None)]
-    polarizationMode: Annotated[Optional[str], Field(None)]
-    polarizationChannels: Annotated[Optional[str], Field(None)]
-    antennaLookDirection: Annotated[Optional[str], Field(None)]
-    minimumIncidenceAngle: Annotated[Optional[float], Field(None)]
-    maximumIncidenceAngle: Annotated[Optional[float], Field(None)]
-    dopplerFrequency: Annotated[Optional[float], Field(None)]
-    incidenceAngleVariation: Annotated[Optional[float], Field(None)]
+    availabilityTime: Annotated[str, Field(None)]
+    acquisitionStation: Annotated[str, Field(None)]
+    acquisitionSubType: Annotated[str, Field(None)]
+    illuminationAzimuthAngle: Annotated[str, Field(None)]
+    illuminationZenithAngle: Annotated[str, Field(None)]
+    illuminationElevationAngle: Annotated[str, Field(None)]
+    polarizationMode: Annotated[str, Field(None)]
+    polarizationChannels: Annotated[str, Field(None)]
+    antennaLookDirection: Annotated[str, Field(None)]
+    minimumIncidenceAngle: Annotated[float, Field(None)]
+    maximumIncidenceAngle: Annotated[float, Field(None)]
+    dopplerFrequency: Annotated[float, Field(None)]
+    incidenceAngleVariation: Annotated[float, Field(None)]
+
+
+class QueryablesDict(UserDict[str, Any]):
+    """Class inheriting from UserDict which contains queryables with their annotated type;
+
+    :param additional_properties: if additional properties (properties not given in EODAG config)
+                                  are allowed
+    :param kwargs: named arguments to initialise the dict (queryable keys + annotated types)
+    """
+
+    additional_properties: bool = Field(True)
+    additional_information: str = Field("")
+
+    def __init__(
+        self,
+        additional_properties: bool = True,
+        additional_information: str = "",
+        **kwargs: Any,
+    ):
+        self.additional_properties = additional_properties
+        self.additional_information = additional_information
+        super().__init__(kwargs)
+
+    def _repr_html_(self, embedded: bool = False) -> str:
+        add_info = (
+            f" additional_information={self.additional_information}"
+            if self.additional_information
+            else ""
+        )
+        thead = (
+            f"""<thead><tr><td style='text-align: left; color: grey;'>
+                {type(self).__name__}&ensp;({len(self)})&ensp;-&ensp;additional_properties={
+                self.additional_properties}
+            """
+            + add_info
+            + "</td></tr></thead>"
+            if not embedded
+            else ""
+        )
+        tr_style = "style='background-color: transparent;'" if embedded else ""
+        return (
+            f"<table>{thead}<tbody>"
+            + "".join(
+                [
+                    f"""<tr {tr_style}><td style='text-align: left;'>
+                        <details><summary style='color: grey;'>
+                        <span style='color: black'>'{k}'</span>:&ensp;
+                        typing.Annotated[{
+                        "<span style='color: black'>" + shorter_type_repr(v.__args__[0]) + "</span>,&ensp;"
+                    }
+                    FieldInfo({"'default': '<span style='color: black'>"
+                               + str(v.__metadata__[0].get_default()) + "</span>',&ensp;"
+                               if v.__metadata__[0].get_default()
+                               and v.__metadata__[0].get_default() != PydanticUndefined else ""}
+                            {"'required': <span style='color: black'>"
+                             + str(v.__metadata__[0].is_required()) + "</span>,"}
+                            ...
+                        )]
+                    </summary>
+                        <span style='color: grey'>typing.Annotated[</span><table style='margin: 0;'>
+                            <tr style='background-color: transparent;'>
+                                <td style='padding: 5px 0 0 10px; text-align: left; vertical-align:top;'>
+                                {remove_class_repr(str(v.__args__[0]))},</td>
+                            </tr><tr style='background-color: transparent;'>
+                                <td style='padding: 5px 0 0 10px; text-align: left; vertical-align:top;'>
+                                {v.__metadata__[0].__repr__()}</td>
+                            </tr>
+                        </table><span style='color: grey'>]</span>
+                    </details>
+                    </td></tr>
+                    """
+                    for k, v in self.items()
+                ]
+            )
+            + "</tbody></table>"
+        )
+
+    def get_model(self, model_name: str = "Queryables") -> BaseModel:
+        """
+        Converts object from :class:`eodag.api.product.QueryablesDict` to :class:`pydantic.BaseModel`
+        so that validation can be performed
+
+        :param model_name: name used for :class:`pydantic.BaseModel` creation
+        :return: pydantic BaseModel of the queryables dict
+        """
+        return annotated_dict_to_model(model_name, self.data)

eodag/types/search_args.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 import re
 from datetime import datetime
-from typing import Annotated, Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Annotated, Any, Optional, Union, cast
 
 from annotated_types import MinLen
 from pydantic import BaseModel, ConfigDict, Field, conint, field_validator
@@ -31,10 +31,10 @@ from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 from eodag.utils.exceptions import ValidationError
 
 NumType = Union[float, int]
-GeomArgs = Union[List[NumType], Tuple[NumType], Dict[str, NumType], str, BaseGeometry]
+GeomArgs = Union[list[NumType], tuple[NumType], dict[str, NumType], str, BaseGeometry]
 
 PositiveInt = conint(gt=0)
-SortByList = Annotated[List[Tuple[str, str]], MinLen(1)]
+SortByList = Annotated[list[tuple[str, str]], MinLen(1)]
 
 
 class SearchArgs(BaseModel):
@@ -48,7 +48,7 @@ class SearchArgs(BaseModel):
     start: Optional[str] = Field(None)
     end: Optional[str] = Field(None)
     geom: Optional[BaseGeometry] = Field(None)
-    locations: Optional[Dict[str, str]] = Field(None)
+    locations: Optional[dict[str, str]] = Field(None)
     page: Optional[int] = Field(DEFAULT_PAGE, gt=0)  # type: ignore
     items_per_page: Optional[PositiveInt] = Field(DEFAULT_ITEMS_PER_PAGE)  # type: ignore
     sort_by: Optional[SortByList] = Field(None)  # type: ignore

eodag/types/whoosh.py

@@ -15,13 +15,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import List
-
 from whoosh.fields import Schema
+from whoosh.index import _DEF_INDEX_NAME, FileIndex
 from whoosh.matching import NullMatcher
 from whoosh.qparser import OrGroup, QueryParser, plugins
 from whoosh.query.positional import Phrase
 from whoosh.query.qcore import QueryError
+from whoosh.util.text import utf8encode
+from whoosh.writing import SegmentWriter
 
 
 class RobustPhrase(Phrase):
@@ -49,7 +50,7 @@ class EODAGQueryParser(QueryParser):
 
     def __init__(
         self,
-        filters: List[str],
+        filters: list[str],
         schema: Schema,
     ):
         """
@@ -77,3 +78,126 @@ class EODAGQueryParser(QueryParser):
             phraseclass=RobustPhrase,
             group=OrGroup,
         )
+
+
+class CleanSegmentWriter(SegmentWriter):
+    """Override to clean up writer for failed document add when exceptions were absorbed
+    cf: https://github.com/whoosh-community/whoosh/pull/543
+    """
+
+    def add_document(self, **fields):
+        """Add document"""
+        self._check_state()
+        perdocwriter = self.perdocwriter
+        schema = self.schema
+        docnum = self.docnum
+        add_post = self.pool.add
+
+        docboost = self._doc_boost(fields)
+        fieldnames = sorted(
+            [name for name in fields.keys() if not name.startswith("_")]
+        )
+        self._check_fields(schema, fieldnames)
+
+        perdocwriter.start_doc(docnum)
+
+        try:
+            for fieldname in fieldnames:
+                value = fields.get(fieldname)
+                if value is None:
+                    continue
+                field = schema[fieldname]
+
+                length = 0
+                if field.indexed:
+                    # TODO: Method for adding progressive field values, ie
+                    # setting start_pos/start_char?
+                    fieldboost = self._field_boost(fields, fieldname, docboost)
+                    # Ask the field to return a list of (text, weight, vbytes)
+                    # tuples
+                    items = field.index(value)
+                    # Only store the length if the field is marked scorable
+                    scorable = field.scorable
+                    # Add the terms to the pool
+                    for tbytes, freq, weight, vbytes in items:
+                        weight *= fieldboost
+                        if scorable:
+                            length += freq
+                        add_post((fieldname, tbytes, docnum, weight, vbytes))
+
+                if field.separate_spelling():
+                    spellfield = field.spelling_fieldname(fieldname)
+                    for word in field.spellable_words(value):
+                        word = utf8encode(word)[0]
+                        add_post((spellfield, word, 0, 1, vbytes))
+
+                vformat = field.vector
+                if vformat:
+                    analyzer = field.analyzer
+                    # Call the format's word_values method to get posting values
+                    vitems = vformat.word_values(value, analyzer, mode="index")
+                    # Remove unused frequency field from the tuple
+                    vitems = sorted(
+                        (text, weight, vbytes) for text, _, weight, vbytes in vitems
+                    )
+                    perdocwriter.add_vector_items(fieldname, field, vitems)
+
+                # Allow a custom value for stored field/column
+                customval = fields.get("_stored_%s" % fieldname, value)
+
+                # Add the stored value and length for this field to the per-
+                # document writer
+                sv = customval if field.stored else None
+                perdocwriter.add_field(fieldname, field, sv, length)
+
+                column = field.column_type
+                if column and customval is not None:
+                    cv = field.to_column_value(customval)
+                    perdocwriter.add_column_value(fieldname, column, cv)
+        except Exception as ex:
+            # cancel doc
+            perdocwriter._doccount -= 1
+            perdocwriter._indoc = False
+            raise ex
+
+        perdocwriter.finish_doc()
+        self._added = True
+        self.docnum += 1
+
+
+class CleanFileIndex(FileIndex):
+    """Override to call CleanSegmentWriter"""
+
+    def writer(self, procs=1, **kwargs):
+        """file index writer"""
+        if procs > 1:
+            from whoosh.multiproc import MpWriter
+
+            return MpWriter(self, procs=procs, **kwargs)
+        else:
+            return CleanSegmentWriter(self, **kwargs)
+
+
+def create_in(dirname, schema, indexname=None):
+    """
+    Override to call the CleanFileIndex.
+
+    Convenience function to create an index in a directory. Takes care of
+    creating a FileStorage object for you.
+
+    :param dirname: the path string of the directory in which to create the
+        index.
+    :param schema: a :class:`whoosh.fields.Schema` object describing the
+        index's fields.
+    :param indexname: the name of the index to create; you only need to specify
+        this if you are creating multiple indexes within the same storage
+        object.
+    :returns: :class:`Index`
+    """
+
+    from whoosh.filedb.filestore import FileStorage
+
+    if not indexname:
+        indexname = _DEF_INDEX_NAME
+    storage = FileStorage(dirname)
+    return CleanFileIndex.create(storage, schema, indexname)