sfeos-helpers 6.8.1__py3-none-any.whl → 6.10.0__py3-none-any.whl

stac_fastapi/sfeos_helpers/mappings.py

@@ -25,11 +25,135 @@ Function Naming Conventions:
 - Parameter names should be consistent across similar functions
 """
 
+import copy
+import json
+import logging
 import os
-from typing import Any, Dict, Literal, Protocol
+from typing import Any, Dict, Literal, Optional, Protocol, Union
 
 from stac_fastapi.core.utilities import get_bool_env
 
+logger = logging.getLogger(__name__)
+
+
+def merge_mappings(base: Dict[str, Any], custom: Dict[str, Any]) -> None:
+    """Recursively merge custom mappings into base mappings.
+
+    Custom mappings will overwrite base mappings if keys collide.
+    Nested dictionaries are merged recursively.
+
+    Args:
+        base: The base mapping dictionary to merge into (modified in place).
+        custom: The custom mapping dictionary to merge from.
+    """
+    for key, value in custom.items():
+        if key in base and isinstance(base[key], dict) and isinstance(value, dict):
+            merge_mappings(base[key], value)
+        else:
+            base[key] = value
+
+
+def parse_dynamic_mapping_config(
+    config_value: Optional[str],
+) -> Union[bool, str]:
+    """Parse the dynamic mapping configuration value.
+
+    Args:
+        config_value: The configuration value from environment variable.
+            Can be "true", "false", "strict", or None.
+
+    Returns:
+        True for "true" (default), False for "false", or the string value
+        for other settings like "strict".
+    """
+    if config_value is None:
+        return True
+    config_lower = config_value.lower()
+    if config_lower == "true":
+        return True
+    elif config_lower == "false":
+        return False
+    else:
+        return config_lower
+
+
+def apply_custom_mappings(
+    mappings: Dict[str, Any], custom_mappings_json: Optional[str]
+) -> None:
+    """Apply custom mappings from a JSON string to the mappings dictionary.
+
+    The custom mappings JSON should have the same structure as ES_ITEMS_MAPPINGS.
+    It will be recursively merged at the root level, allowing users to override
+    any part of the mapping including properties, dynamic_templates, etc.
+
+    Args:
+        mappings: The mappings dictionary to modify (modified in place).
+        custom_mappings_json: JSON string containing custom mappings.
+
+    Raises:
+        Logs error if JSON parsing or merging fails.
+    """
+    if not custom_mappings_json:
+        return
+
+    try:
+        custom_mappings = json.loads(custom_mappings_json)
+        merge_mappings(mappings, custom_mappings)
+    except json.JSONDecodeError as e:
+        logger.error(f"Failed to parse STAC_FASTAPI_ES_CUSTOM_MAPPINGS JSON: {e}")
+    except Exception as e:
+        logger.error(f"Failed to merge STAC_FASTAPI_ES_CUSTOM_MAPPINGS: {e}")
+
+
+def get_items_mappings(
+    dynamic_mapping: Optional[str] = None, custom_mappings: Optional[str] = None
+) -> Dict[str, Any]:
+    """Get the ES_ITEMS_MAPPINGS with optional dynamic mapping and custom mappings applied.
+
+    This function creates a fresh copy of the base mappings and applies the
+    specified configuration. Useful for testing or programmatic configuration.
+
+    Args:
+        dynamic_mapping: Override for STAC_FASTAPI_ES_DYNAMIC_MAPPING.
+            If None, reads from environment variable.
+        custom_mappings: Override for STAC_FASTAPI_ES_CUSTOM_MAPPINGS.
+            If None, reads from environment variable.
+
+    Returns:
+        A new dictionary containing the configured mappings.
+    """
+    mappings = copy.deepcopy(_BASE_ITEMS_MAPPINGS)
+
+    # Apply dynamic mapping configuration
+    dynamic_config = (
+        dynamic_mapping
+        if dynamic_mapping is not None
+        else os.getenv("STAC_FASTAPI_ES_DYNAMIC_MAPPING", "true")
+    )
+    mappings["dynamic"] = parse_dynamic_mapping_config(dynamic_config)
+
+    # Apply custom mappings
+    custom_config = (
+        custom_mappings
+        if custom_mappings is not None
+        else os.getenv("STAC_FASTAPI_ES_CUSTOM_MAPPINGS")
+    )
+
+    if custom_config is None:
+        mappings_file = os.getenv("STAC_FASTAPI_ES_MAPPINGS_FILE")
+        if mappings_file:
+            try:
+                with open(mappings_file, "r") as f:
+                    custom_config = f.read()
+            except Exception as e:
+                logger.error(
+                    f"Failed to read STAC_FASTAPI_ES_MAPPINGS_FILE at {mappings_file}: {e}"
+                )
+
+    apply_custom_mappings(mappings, custom_config)
+
+    return mappings
+
 
 # stac_pydantic classes extend _GeometryBase, which doesn't have a type field,
 # So create our own Protocol for typing
@@ -129,7 +253,8 @@ ES_MAPPINGS_DYNAMIC_TEMPLATES = [
     },
 ]
 
-ES_ITEMS_MAPPINGS = {
+# Base items mappings without dynamic configuration applied
+_BASE_ITEMS_MAPPINGS = {
     "numeric_detection": False,
     "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,
     "properties": {
@@ -143,8 +268,8 @@ ES_ITEMS_MAPPINGS = {
             "properties": {
                 # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
                 "datetime": {"type": "date_nanos"},
-                "start_datetime": {"type": "date"},
-                "end_datetime": {"type": "date"},
+                "start_datetime": {"type": "date_nanos"},
+                "end_datetime": {"type": "date_nanos"},
                 "created": {"type": "date"},
                 "updated": {"type": "date"},
                 # Satellite Extension https://github.com/stac-extensions/sat
@@ -155,6 +280,9 @@ ES_ITEMS_MAPPINGS = {
     },
 }
 
+# ES_ITEMS_MAPPINGS with environment-based configuration applied at module load time
+ES_ITEMS_MAPPINGS = get_items_mappings()
+
 ES_COLLECTIONS_MAPPINGS = {
     "numeric_detection": False,
     "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,

stac_fastapi/sfeos_helpers/models/patch.py

@@ -3,7 +3,7 @@
 import re
 from typing import Any, Dict, Optional, Union
 
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, ConfigDict, model_validator
 
 regex = re.compile(r"([^.' ]*:[^.'[ ]*)\.?")
 replacements = str.maketrans({"/": "", ".": "", ":": "", "[": "", "]": ""})
@@ -84,10 +84,7 @@ class ElasticPath(BaseModel):
     variable_name: Optional[str] = None
     param_key: Optional[str] = None
 
-    class Config:
-        """Class config."""
-
-        frozen = True
+    model_config = ConfigDict(frozen=True)
 
     @model_validator(mode="before")
     @classmethod

stac_fastapi/sfeos_helpers/search_engine/base.py

@@ -49,3 +49,33 @@ class BaseIndexInserter(ABC):
             str: Created index name.
         """
         pass
+
+    @staticmethod
+    @abstractmethod
+    def should_create_collection_index() -> bool:
+        """Whether this strategy requires collection index creation.
+
+        Returns:
+            bool: True if strategy creates collection indexes, False otherwise.
+        """
+        pass
+
+    async def refresh_cache(self) -> None:
+        """Refresh internal cache if applicable.
+
+        Default implementation does nothing. Subclasses that maintain
+        internal caches should override this method.
+        """
+        pass
+
+    def validate_datetime_field_update(self, field_path: str) -> None:
+        """Validate if a datetime field can be updated.
+
+        For datetime-based indexing, certain datetime fields cannot be modified
+        because they determine the index where the item is stored.
+
+        Args:
+            field_path (str): The path of the field being updated (e.g., "properties.datetime").
+
+        """
+        pass

stac_fastapi/sfeos_helpers/search_engine/index_operations.py

@@ -1,8 +1,9 @@
 """Search engine adapters for different implementations."""
 
 import uuid
-from typing import Any, Dict
+from typing import Any, Dict, List, Literal
 
+from stac_fastapi.core.utilities import get_bool_env
 from stac_fastapi.sfeos_helpers.database import (
     index_alias_by_collection_id,
     index_by_collection_id,
@@ -18,6 +19,16 @@ from stac_fastapi.sfeos_helpers.mappings import (
 class IndexOperations:
     """Base class for search engine adapters with common implementations."""
 
+    @property
+    def use_datetime(self) -> bool:
+        """Get USE_DATETIME setting dynamically."""
+        return get_bool_env("USE_DATETIME", default=True)
+
+    @property
+    def primary_datetime_name(self) -> str:
+        """Get primary datetime field name based on current USE_DATETIME setting."""
+        return "datetime" if self.use_datetime else "start_datetime"
+
     async def create_simple_index(self, client: Any, collection_id: str) -> str:
         """Create a simple index for the given collection.
 
@@ -31,34 +42,65 @@ class IndexOperations:
         index_name = f"{index_by_collection_id(collection_id)}-000001"
         alias_name = index_alias_by_collection_id(collection_id)
 
-        await client.indices.create(
-            index=index_name,
-            body=self._create_index_body({alias_name: {}}),
-            params={"ignore": [400]},
-        )
+        if hasattr(client, "options"):
+            await client.options(ignore_status=[400]).indices.create(
+                index=index_name,
+                body=self._create_index_body({alias_name: {}}),
+            )
+        else:
+            await client.indices.create(
+                index=index_name,
+                body=self._create_index_body({alias_name: {}}),
+                params={"ignore": [400]},
+            )
         return index_name
 
     async def create_datetime_index(
-        self, client: Any, collection_id: str, start_date: str
+        self,
+        client: Any,
+        collection_id: str,
+        start_datetime: str | None,
+        datetime: str | None,
+        end_datetime: str | None,
     ) -> str:
         """Create a datetime-based index for the given collection.
 
         Args:
             client: Search engine client instance.
             collection_id (str): Collection identifier.
-            start_date (str): Start date for the alias.
+            start_datetime (str | None): Start datetime for the index alias.
+            datetime (str | None): Datetime for the datetime alias.
+            end_datetime (str | None): End datetime for the index alias.
 
         Returns:
-            str: Created index alias name.
+            str: Created datetime alias name.
         """
         index_name = self.create_index_name(collection_id)
-        alias_name = self.create_alias_name(collection_id, start_date)
         collection_alias = index_alias_by_collection_id(collection_id)
+
+        aliases: Dict[str, Any] = {
+            collection_alias: {},
+        }
+
+        if start_datetime:
+            alias_start_date = self.create_alias_name(
+                collection_id, "start_datetime", start_datetime
+            )
+            alias_end_date = self.create_alias_name(
+                collection_id, "end_datetime", end_datetime
+            )
+            aliases[alias_start_date] = {}
+            aliases[alias_end_date] = {}
+            created_alias = alias_start_date
+        else:
+            created_alias = self.create_alias_name(collection_id, "datetime", datetime)
+            aliases[created_alias] = {}
+
         await client.indices.create(
             index=index_name,
-            body=self._create_index_body({collection_alias: {}, alias_name: {}}),
+            body=self._create_index_body(aliases),
         )
-        return alias_name
+        return created_alias
 
     @staticmethod
     async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str:
@@ -84,23 +126,33 @@ class IndexOperations:
         return new_alias
 
     @staticmethod
-    async def change_alias_name(client: Any, old_alias: str, new_alias: str) -> None:
-        """Change alias name from old to new.
+    async def change_alias_name(
+        client: Any,
+        old_start_datetime_alias: str,
+        aliases_to_change: List[str],
+        aliases_to_create: List[str],
+    ) -> None:
+        """Change alias names by removing old aliases and adding new ones.
 
         Args:
             client: Search engine client instance.
-            old_alias (str): Current alias name.
-            new_alias (str): New alias name.
+            old_start_datetime_alias (str): Current start_datetime alias name to identify the index.
+            aliases_to_change (List[str]): List of old alias names to remove.
+            aliases_to_create (List[str]): List of new alias names to add.
 
         Returns:
             None
         """
-        aliases_info = await client.indices.get_alias(name=old_alias)
-        actions = []
+        aliases_info = await client.indices.get_alias(name=old_start_datetime_alias)
+        index_name = list(aliases_info.keys())[0]
 
-        for index_name in aliases_info.keys():
+        actions = []
+        for old_alias in aliases_to_change:
             actions.append({"remove": {"index": index_name, "alias": old_alias}})
+
+        for new_alias in aliases_to_create:
             actions.append({"add": {"index": index_name, "alias": new_alias}})
+
         await client.indices.update_aliases(body={"actions": actions})
 
     @staticmethod
@@ -117,18 +169,23 @@ class IndexOperations:
         return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{uuid.uuid4()}"
 
     @staticmethod
-    def create_alias_name(collection_id: str, start_date: str) -> str:
-        """Create index name from collection ID and uuid4.
+    def create_alias_name(
+        collection_id: str,
+        name: Literal["start_datetime", "datetime", "end_datetime"],
+        start_date: str,
+    ) -> str:
+        """Create alias name from collection ID and date.
 
         Args:
             collection_id (str): Collection identifier.
-            start_date (str): Start date for the alias.
+            name (Literal["start_datetime", "datetime", "end_datetime"]): Type of alias to create.
+            start_date (str): Date value for the alias.
 
         Returns:
-            str: Alias name with initial date.
+            str: Formatted alias name with prefix, type, collection ID, and date.
         """
         cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
-        return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{start_date}"
+        return f"{ITEMS_INDEX_PREFIX}{name}_{cleaned.lower()}_{start_date}"
 
     @staticmethod
     def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]:
@@ -146,21 +203,25 @@ class IndexOperations:
             "settings": ES_ITEMS_SETTINGS,
         }
 
-    @staticmethod
-    async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, Any]:
-        """Find the latest item date in the specified index.
+    async def find_latest_item_in_index(
+        self, client: Any, index_name: str
+    ) -> dict[str, Any]:
+        """Find the latest item in the specified index.
 
         Args:
             client: Search engine client instance.
             index_name (str): Name of the index to query.
 
         Returns:
-            datetime: Date of the latest item in the index.
+            dict[str, Any]: Latest item document from the index with metadata.
         """
         query = {
             "size": 1,
-            "sort": [{"properties.datetime": {"order": "desc"}}],
-            "_source": ["properties.datetime"],
+            "sort": [{f"properties.{self.primary_datetime_name}": {"order": "desc"}}],
+            "_source": [
+                "properties.start_datetime",
+                "properties.datetime",
+            ],
         }
 
         response = await client.search(index=index_name, body=query)