eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/types/search_args.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 import re
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Annotated, Any, Optional, Union, cast
 
 from annotated_types import MinLen
 from pydantic import BaseModel, ConfigDict, Field, conint, field_validator
@@ -27,14 +27,14 @@ from shapely.geometry import Polygon, shape
 from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
 
 from eodag.types.bbox import BBox
-from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, Annotated
+from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 from eodag.utils.exceptions import ValidationError
 
 NumType = Union[float, int]
-GeomArgs = Union[List[NumType], Tuple[NumType], Dict[str, NumType], str, BaseGeometry]
+GeomArgs = Union[list[NumType], tuple[NumType], dict[str, NumType], str, BaseGeometry]
 
 PositiveInt = conint(gt=0)
-SortByList = Annotated[List[Tuple[str, str]], MinLen(1)]
+SortByList = Annotated[list[tuple[str, str]], MinLen(1)]
 
 
 class SearchArgs(BaseModel):
@@ -48,7 +48,7 @@ class SearchArgs(BaseModel):
     start: Optional[str] = Field(None)
     end: Optional[str] = Field(None)
     geom: Optional[BaseGeometry] = Field(None)
-    locations: Optional[Dict[str, str]] = Field(None)
+    locations: Optional[dict[str, str]] = Field(None)
     page: Optional[int] = Field(DEFAULT_PAGE, gt=0)  # type: ignore
     items_per_page: Optional[PositiveInt] = Field(DEFAULT_ITEMS_PER_PAGE)  # type: ignore
     sort_by: Optional[SortByList] = Field(None)  # type: ignore
@@ -119,7 +119,7 @@ class SearchArgs(BaseModel):
             )
             sort_by_arg[i] = (sort_param, sort_order[:3])
         # remove duplicates
-        pruned_sort_by_arg: SortByList = list(set(sort_by_arg))  # type: ignore
+        pruned_sort_by_arg: SortByList = list(dict.fromkeys(sort_by_arg))  # type: ignore
         for i, sort_by_tuple in enumerate(pruned_sort_by_arg):
             for j, sort_by_tuple_tmp in enumerate(pruned_sort_by_arg):
                 # since duplicated tuples or dictionnaries have been removed, if two sorting parameters are equal,

eodag/types/whoosh.py

@@ -15,13 +15,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import List
-
 from whoosh.fields import Schema
+from whoosh.index import _DEF_INDEX_NAME, FileIndex
 from whoosh.matching import NullMatcher
 from whoosh.qparser import OrGroup, QueryParser, plugins
 from whoosh.query.positional import Phrase
 from whoosh.query.qcore import QueryError
+from whoosh.util.text import utf8encode
+from whoosh.writing import SegmentWriter
 
 
 class RobustPhrase(Phrase):
@@ -49,7 +50,7 @@ class EODAGQueryParser(QueryParser):
 
     def __init__(
         self,
-        filters: List[str],
+        filters: list[str],
         schema: Schema,
     ):
         """
@@ -77,3 +78,126 @@ class EODAGQueryParser(QueryParser):
             phraseclass=RobustPhrase,
             group=OrGroup,
         )
+
+
+class CleanSegmentWriter(SegmentWriter):
+    """Override to clean up writer for failed document add when exceptions were absorbed
+    cf: https://github.com/whoosh-community/whoosh/pull/543
+    """
+
+    def add_document(self, **fields):
+        """Add document"""
+        self._check_state()
+        perdocwriter = self.perdocwriter
+        schema = self.schema
+        docnum = self.docnum
+        add_post = self.pool.add
+
+        docboost = self._doc_boost(fields)
+        fieldnames = sorted(
+            [name for name in fields.keys() if not name.startswith("_")]
+        )
+        self._check_fields(schema, fieldnames)
+
+        perdocwriter.start_doc(docnum)
+
+        try:
+            for fieldname in fieldnames:
+                value = fields.get(fieldname)
+                if value is None:
+                    continue
+                field = schema[fieldname]
+
+                length = 0
+                if field.indexed:
+                    # TODO: Method for adding progressive field values, ie
+                    # setting start_pos/start_char?
+                    fieldboost = self._field_boost(fields, fieldname, docboost)
+                    # Ask the field to return a list of (text, weight, vbytes)
+                    # tuples
+                    items = field.index(value)
+                    # Only store the length if the field is marked scorable
+                    scorable = field.scorable
+                    # Add the terms to the pool
+                    for tbytes, freq, weight, vbytes in items:
+                        weight *= fieldboost
+                        if scorable:
+                            length += freq
+                        add_post((fieldname, tbytes, docnum, weight, vbytes))
+
+                if field.separate_spelling():
+                    spellfield = field.spelling_fieldname(fieldname)
+                    for word in field.spellable_words(value):
+                        word = utf8encode(word)[0]
+                        add_post((spellfield, word, 0, 1, vbytes))
+
+                vformat = field.vector
+                if vformat:
+                    analyzer = field.analyzer
+                    # Call the format's word_values method to get posting values
+                    vitems = vformat.word_values(value, analyzer, mode="index")
+                    # Remove unused frequency field from the tuple
+                    vitems = sorted(
+                        (text, weight, vbytes) for text, _, weight, vbytes in vitems
+                    )
+                    perdocwriter.add_vector_items(fieldname, field, vitems)
+
+                # Allow a custom value for stored field/column
+                customval = fields.get("_stored_%s" % fieldname, value)
+
+                # Add the stored value and length for this field to the per-
+                # document writer
+                sv = customval if field.stored else None
+                perdocwriter.add_field(fieldname, field, sv, length)
+
+                column = field.column_type
+                if column and customval is not None:
+                    cv = field.to_column_value(customval)
+                    perdocwriter.add_column_value(fieldname, column, cv)
+        except Exception as ex:
+            # cancel doc
+            perdocwriter._doccount -= 1
+            perdocwriter._indoc = False
+            raise ex
+
+        perdocwriter.finish_doc()
+        self._added = True
+        self.docnum += 1
+
+
+class CleanFileIndex(FileIndex):
+    """Override to call CleanSegmentWriter"""
+
+    def writer(self, procs=1, **kwargs):
+        """file index writer"""
+        if procs > 1:
+            from whoosh.multiproc import MpWriter
+
+            return MpWriter(self, procs=procs, **kwargs)
+        else:
+            return CleanSegmentWriter(self, **kwargs)
+
+
+def create_in(dirname, schema, indexname=None):
+    """
+    Override to call the CleanFileIndex.
+
+    Convenience function to create an index in a directory. Takes care of
+    creating a FileStorage object for you.
+
+    :param dirname: the path string of the directory in which to create the
+        index.
+    :param schema: a :class:`whoosh.fields.Schema` object describing the
+        index's fields.
+    :param indexname: the name of the index to create; you only need to specify
+        this if you are creating multiple indexes within the same storage
+        object.
+    :returns: :class:`Index`
+    """
+
+    from whoosh.filedb.filestore import FileStorage
+
+    if not indexname:
+        indexname = _DEF_INDEX_NAME
+    storage = FileStorage(dirname)
+    return CleanFileIndex.create(storage, schema, indexname)