sfeos-helpers 6.1.0__py3-none-any.whl → 6.2.1__py3-none-any.whl

stac_fastapi/sfeos_helpers/models/patch.py

@@ -3,7 +3,7 @@
 import re
 from typing import Any, Dict, Optional, Union
 
-from pydantic import BaseModel, computed_field, model_validator
+from pydantic import BaseModel, model_validator
 
 regex = re.compile(r"([^.' ]*:[^.'[ ]*)\.?")
 replacements = str.maketrans({"/": "", ".": "", ":": "", "[": "", "]": ""})
@@ -71,16 +71,23 @@ class ElasticPath(BaseModel):
 
     """
 
-    path: str
+    parts: list[str] = []
+
+    path: Optional[str] = None
+    key: Optional[Union[str, int]] = None
     nest: Optional[str] = None
-    partition: Optional[str] = None
-    key: Optional[str] = None
 
     es_path: Optional[str] = None
-    es_nest: Optional[str] = None
     es_key: Optional[str] = None
+    es_nest: Optional[str] = None
+
+    variable_name: Optional[str] = None
+    param_key: Optional[str] = None
 
-    index_: Optional[int] = None
+    class Config:
+        """Class config."""
+
+        frozen = True
 
     @model_validator(mode="before")
     @classmethod
@@ -90,77 +97,28 @@ class ElasticPath(BaseModel):
         Args:
             data (Any): input data
         """
-        data["path"] = data["path"].lstrip("/").replace("/", ".")
-        data["nest"], data["partition"], data["key"] = data["path"].rpartition(".")
-
-        if data["key"].lstrip("-").isdigit() or data["key"] == "-":
-            data["index_"] = -1 if data["key"] == "-" else int(data["key"])
-            data["path"] = f"{data['nest']}[{data['index_']}]"
-            data["nest"], data["partition"], data["key"] = data["nest"].rpartition(".")
-
-        data["es_path"] = to_es(data["path"])
-        data["es_nest"] = f".{to_es(data['nest'])}" if data["nest"] else ""
-        data["es_key"] = to_es(data["key"])
-
-        return data
-
-    @computed_field  # type: ignore[misc]
-    @property
-    def index(self) -> Union[int, str, None]:
-        """Compute location of path.
-
-        Returns:
-            str: path index
-        """
-        if self.index_ and self.index_ < 0:
+        data["parts"] = data["path"].lstrip("/").split("/")
 
-            return f"ctx._source.{self.location}.size() - {-self.index_}"
+        data["key"] = data["parts"].pop(-1)
+        data["nest"] = "/".join(data["parts"])
+        data["path"] = data["nest"] + "/" + data["key"]
 
-        return self.index_
+        data["es_key"] = data["key"]
+        data["es_nest"] = "".join([f"['{part}']" for part in data["parts"]])
+        data["es_path"] = data["es_nest"] + f"['{data['es_key']}']"
 
-    @computed_field  # type: ignore[misc]
-    @property
-    def location(self) -> str:
-        """Compute location of path.
-
-        Returns:
-            str: path location
-        """
-        return self.nest + self.partition + self.key
-
-    @computed_field  # type: ignore[misc]
-    @property
-    def es_location(self) -> str:
-        """Compute location of path.
-
-        Returns:
-            str: path location
-        """
-        if self.es_key and ":" in self.es_key:
-            return self.es_nest + self.es_key
-        return self.es_nest + self.partition + self.es_key
-
-    @computed_field  # type: ignore[misc]
-    @property
-    def variable_name(self) -> str:
-        """Variable name for scripting.
-
-        Returns:
-            str: variable name
-        """
-        if self.index is not None:
-            return f"{self.location.replace('.','_').replace(':','_')}_{self.index}"
-
-        return (
-            f"{self.nest.replace('.','_').replace(':','_')}_{self.key.replace(':','_')}"
-        )
-
-    @computed_field  # type: ignore[misc]
-    @property
-    def param_key(self) -> str:
-        """Param key for scripting.
+        if data["key"].lstrip("-").isdigit() or data["key"] == "-":
+            data["key"] = -1 if data["key"] == "-" else int(data["key"])
+            data["es_key"] = (
+                f"ctx._source{data['es_nest']}.size() - {-data['key']}"
+                if data["key"] < 0
+                else str(data["key"])
+            )
+            data["es_path"] = data["es_nest"] + f"[{data['es_key']}]"
+
+        data[
+            "variable_name"
+        ] = f"{data['nest'].replace('/','_').replace(':','_')}_{str(data['key']).replace(':','_')}"
+        data["param_key"] = data["path"].translate(replacements)
 
-        Returns:
-            str: param key
-        """
-        return self.path.translate(replacements)
+        return data

stac_fastapi/sfeos_helpers/search_engine/__init__.py

@@ -0,0 +1,27 @@
+"""Search engine index management package."""
+
+from .base import BaseIndexInserter
+from .factory import IndexInsertionFactory
+from .index_operations import IndexOperations
+from .inserters import DatetimeIndexInserter, SimpleIndexInserter
+from .managers import DatetimeIndexManager, IndexSizeManager
+from .selection import (
+    BaseIndexSelector,
+    DatetimeBasedIndexSelector,
+    IndexSelectorFactory,
+    UnfilteredIndexSelector,
+)
+
+__all__ = [
+    "BaseIndexInserter",
+    "BaseIndexSelector",
+    "IndexOperations",
+    "IndexSizeManager",
+    "DatetimeIndexManager",
+    "DatetimeIndexInserter",
+    "SimpleIndexInserter",
+    "IndexInsertionFactory",
+    "DatetimeBasedIndexSelector",
+    "UnfilteredIndexSelector",
+    "IndexSelectorFactory",
+]

stac_fastapi/sfeos_helpers/search_engine/base.py

@@ -0,0 +1,51 @@
+"""Base classes for index inserters."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+
+class BaseIndexInserter(ABC):
+    """Base async index inserter with common async methods."""
+
+    @abstractmethod
+    async def get_target_index(
+        self, collection_id: str, product: Dict[str, Any]
+    ) -> str:
+        """Get target index for a product asynchronously.
+
+        Args:
+            collection_id (str): Collection identifier.
+            product (Dict[str, Any]): Product data.
+
+        Returns:
+            str: Target index name.
+        """
+        pass
+
+    @abstractmethod
+    async def prepare_bulk_actions(
+        self, collection_id: str, items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Prepare bulk actions for multiple items asynchronously.
+
+        Args:
+            collection_id (str): Collection identifier.
+            items (List[Dict[str, Any]]): List of items to process.
+
+        Returns:
+            List[Dict[str, Any]]: List of bulk actions.
+        """
+        pass
+
+    @abstractmethod
+    async def create_simple_index(self, client: Any, collection_id: str) -> str:
+        """Create a simple index asynchronously.
+
+        Args:
+            client: Search engine client instance.
+            collection_id (str): Collection identifier.
+
+        Returns:
+            str: Created index name.
+        """
+        pass

stac_fastapi/sfeos_helpers/search_engine/factory.py

@@ -0,0 +1,36 @@
+"""Factory for creating index insertion strategies."""
+
+from typing import Any
+
+from stac_fastapi.core.utilities import get_bool_env
+
+from .base import BaseIndexInserter
+from .index_operations import IndexOperations
+from .inserters import DatetimeIndexInserter, SimpleIndexInserter
+
+
+class IndexInsertionFactory:
+    """Factory for creating index insertion strategies."""
+
+    @staticmethod
+    def create_insertion_strategy(
+        client: Any,
+    ) -> BaseIndexInserter:
+        """Create async insertion strategy based on configuration.
+
+        Args:
+            client: Async search engine client instance.
+
+        Returns:
+            BaseIndexInserter: Configured async insertion strategy.
+        """
+        index_operations = IndexOperations()
+
+        use_datetime_partitioning = get_bool_env(
+            "ENABLE_DATETIME_INDEX_FILTERING", default="false"
+        )
+
+        if use_datetime_partitioning:
+            return DatetimeIndexInserter(client, index_operations)
+        else:
+            return SimpleIndexInserter(index_operations, client)

stac_fastapi/sfeos_helpers/search_engine/index_operations.py

@@ -0,0 +1,167 @@
+"""Search engine adapters for different implementations."""
+
+import uuid
+from typing import Any, Dict
+
+from stac_fastapi.sfeos_helpers.database import (
+    index_alias_by_collection_id,
+    index_by_collection_id,
+)
+from stac_fastapi.sfeos_helpers.mappings import (
+    _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
+    ES_ITEMS_MAPPINGS,
+    ES_ITEMS_SETTINGS,
+    ITEMS_INDEX_PREFIX,
+)
+
+
+class IndexOperations:
+    """Base class for search engine adapters with common implementations."""
+
+    async def create_simple_index(self, client: Any, collection_id: str) -> str:
+        """Create a simple index for the given collection.
+
+        Args:
+            client: Search engine client instance.
+            collection_id (str): Collection identifier.
+
+        Returns:
+            str: Created index name.
+        """
+        index_name = f"{index_by_collection_id(collection_id)}-000001"
+        alias_name = index_alias_by_collection_id(collection_id)
+
+        await client.indices.create(
+            index=index_name,
+            body=self._create_index_body({alias_name: {}}),
+            params={"ignore": [400]},
+        )
+        return index_name
+
+    async def create_datetime_index(
+        self, client: Any, collection_id: str, start_date: str
+    ) -> str:
+        """Create a datetime-based index for the given collection.
+
+        Args:
+            client: Search engine client instance.
+            collection_id (str): Collection identifier.
+            start_date (str): Start date for the alias.
+
+        Returns:
+            str: Created index alias name.
+        """
+        index_name = self.create_index_name(collection_id)
+        alias_name = self.create_alias_name(collection_id, start_date)
+        collection_alias = index_alias_by_collection_id(collection_id)
+        await client.indices.create(
+            index=index_name,
+            body=self._create_index_body({collection_alias: {}, alias_name: {}}),
+        )
+        return alias_name
+
+    @staticmethod
+    async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str:
+        """Update index alias with new end date.
+
+        Args:
+            client: Search engine client instance.
+            end_date (str): End date for the alias.
+            old_alias (str): Current alias name.
+
+        Returns:
+            str: New alias name.
+        """
+        new_alias = f"{old_alias}-{end_date}"
+        aliases_info = await client.indices.get_alias(name=old_alias)
+        actions = []
+
+        for index_name in aliases_info.keys():
+            actions.append({"remove": {"index": index_name, "alias": old_alias}})
+            actions.append({"add": {"index": index_name, "alias": new_alias}})
+
+        await client.indices.update_aliases(body={"actions": actions})
+        return new_alias
+
+    @staticmethod
+    async def change_alias_name(client: Any, old_alias: str, new_alias: str) -> None:
+        """Change alias name from old to new.
+
+        Args:
+            client: Search engine client instance.
+            old_alias (str): Current alias name.
+            new_alias (str): New alias name.
+
+        Returns:
+            None
+        """
+        aliases_info = await client.indices.get_alias(name=old_alias)
+        actions = []
+
+        for index_name in aliases_info.keys():
+            actions.append({"remove": {"index": index_name, "alias": old_alias}})
+            actions.append({"add": {"index": index_name, "alias": new_alias}})
+        await client.indices.update_aliases(body={"actions": actions})
+
+    @staticmethod
+    def create_index_name(collection_id: str) -> str:
+        """Create index name from collection ID and uuid4.
+
+        Args:
+            collection_id (str): Collection identifier.
+
+        Returns:
+            str: Formatted index name.
+        """
+        cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+        return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{uuid.uuid4()}"
+
+    @staticmethod
+    def create_alias_name(collection_id: str, start_date: str) -> str:
+        """Create index name from collection ID and uuid4.
+
+        Args:
+            collection_id (str): Collection identifier.
+            start_date (str): Start date for the alias.
+
+        Returns:
+            str: Alias name with initial date.
+        """
+        cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+        return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{start_date}"
+
+    @staticmethod
+    def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]:
+        """Create index body with common settings.
+
+        Args:
+            aliases (Dict[str, Dict]): Aliases configuration.
+
+        Returns:
+            Dict[str, Any]: Index body configuration.
+        """
+        return {
+            "aliases": aliases,
+            "mappings": ES_ITEMS_MAPPINGS,
+            "settings": ES_ITEMS_SETTINGS,
+        }
+
+    @staticmethod
+    async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, Any]:
+        """Find the latest item date in the specified index.
+
+        Args:
+            client: Search engine client instance.
+            index_name (str): Name of the index to query.
+
+        Returns:
+            datetime: Date of the latest item in the index.
+        """
+        query = {
+            "size": 1,
+            "sort": [{"properties.datetime": {"order": "desc"}}],
+            "_source": ["properties.datetime"],
+        }
+
+        response = await client.search(index=index_name, body=query)
+        return response["hits"]["hits"][0]