stac-fastapi-opensearch 6.1.0__py3-none-any.whl → 6.2.0__py3-none-any.whl

stac_fastapi/opensearch/app.py

@@ -118,7 +118,7 @@ post_request_model = create_post_request_model(search_extensions)
 app_config = {
     "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"),
     "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"),
-    "api_version": os.getenv("STAC_FASTAPI_VERSION", "6.1.0"),
+    "api_version": os.getenv("STAC_FASTAPI_VERSION", "6.2.0"),
     "settings": settings,
     "extensions": extensions,
     "client": CoreClient(

stac_fastapi/opensearch/config.py

@@ -1,4 +1,5 @@
 """API configuration."""
+
 import logging
 import os
 import ssl

stac_fastapi/opensearch/database_logic.py

@@ -4,7 +4,7 @@ import asyncio
 import logging
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
 
 import attr
 import orjson
@@ -26,7 +26,7 @@ from stac_fastapi.opensearch.config import (
     AsyncOpensearchSettings as AsyncSearchSettings,
 )
 from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
-from stac_fastapi.sfeos_helpers import filter
+from stac_fastapi.sfeos_helpers import filter as filter_module
 from stac_fastapi.sfeos_helpers.database import (
     apply_free_text_filter_shared,
     apply_intersects_filter_shared,
@@ -34,8 +34,6 @@ from stac_fastapi.sfeos_helpers.database import (
     delete_item_index_shared,
     get_queryables_mapping_shared,
     index_alias_by_collection_id,
-    index_by_collection_id,
-    indices,
     mk_actions,
     mk_item_id,
     populate_sort_shared,
@@ -55,15 +53,18 @@ from stac_fastapi.sfeos_helpers.mappings import (
     COLLECTIONS_INDEX,
     DEFAULT_SORT,
     ES_COLLECTIONS_MAPPINGS,
-    ES_ITEMS_MAPPINGS,
-    ES_ITEMS_SETTINGS,
     ITEM_INDICES,
     ITEMS_INDEX_PREFIX,
     Geometry,
 )
+from stac_fastapi.sfeos_helpers.search_engine import (
+    BaseIndexInserter,
+    BaseIndexSelector,
+    IndexInsertionFactory,
+    IndexSelectorFactory,
+)
 from stac_fastapi.types.errors import ConflictError, NotFoundError
 from stac_fastapi.types.links import resolve_links
-from stac_fastapi.types.rfc3339 import DateTimeType
 from stac_fastapi.types.stac import Collection, Item
 
 logger = logging.getLogger(__name__)
@@ -104,33 +105,6 @@ async def create_collection_index() -> None:
     await client.close()
 
 
-async def create_item_index(collection_id: str) -> None:
-    """
-    Create the index for Items. The settings of the index template will be used implicitly.
-
-    Args:
-        collection_id (str): Collection identifier.
-
-    Returns:
-        None
-
-    """
-    client = AsyncSearchSettings().create_client
-
-    index_name = f"{index_by_collection_id(collection_id)}-000001"
-    exists = await client.indices.exists(index=index_name)
-    if not exists:
-        await client.indices.create(
-            index=index_name,
-            body={
-                "aliases": {index_alias_by_collection_id(collection_id): {}},
-                "mappings": ES_ITEMS_MAPPINGS,
-                "settings": ES_ITEMS_SETTINGS,
-            },
-        )
-    await client.close()
-
-
 async def delete_item_index(collection_id: str) -> None:
     """Delete the index for items in a collection.
 
@@ -152,6 +126,9 @@ class DatabaseLogic(BaseDatabaseLogic):
     async_settings: AsyncSearchSettings = attr.ib(factory=AsyncSearchSettings)
     sync_settings: SyncSearchSettings = attr.ib(factory=SyncSearchSettings)
 
+    async_index_selector: BaseIndexSelector = attr.ib(init=False)
+    async_index_inserter: BaseIndexInserter = attr.ib(init=False)
+
     client = attr.ib(init=False)
     sync_client = attr.ib(init=False)
 
@@ -159,6 +136,10 @@ class DatabaseLogic(BaseDatabaseLogic):
         """Initialize clients after the class is instantiated."""
         self.client = self.async_settings.create_client
         self.sync_client = self.sync_settings.create_client
+        self.async_index_inserter = IndexInsertionFactory.create_insertion_strategy(
+            self.client
+        )
+        self.async_index_selector = IndexSelectorFactory.create_selector(self.client)
 
     item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
     collection_serializer: Type[CollectionSerializer] = attr.ib(
@@ -234,15 +215,23 @@ class DatabaseLogic(BaseDatabaseLogic):
             with the index for the Collection as the target index and the combined `mk_item_id` as the document id.
         """
         try:
-            item = await self.client.get(
+            response = await self.client.search(
                 index=index_alias_by_collection_id(collection_id),
-                id=mk_item_id(item_id, collection_id),
+                body={
+                    "query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
+                    "size": 1,
+                },
             )
+            if response["hits"]["total"]["value"] == 0:
+                raise NotFoundError(
+                    f"Item {item_id} does not exist inside Collection {collection_id}"
+                )
+
+            return response["hits"]["hits"][0]["_source"]
         except exceptions.NotFoundError:
             raise NotFoundError(
                 f"Item {item_id} does not exist inside Collection {collection_id}"
             )
-        return item["_source"]
 
     async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
         """Retrieve mapping of Queryables for search.
@@ -296,31 +285,21 @@ class DatabaseLogic(BaseDatabaseLogic):
 
     @staticmethod
     def apply_datetime_filter(
-        search: Search, interval: Optional[Union[DateTimeType, str]]
-    ) -> Search:
+        search: Search, datetime: Optional[str]
+    ) -> Tuple[Search, Dict[str, Optional[str]]]:
         """Apply a filter to search on datetime, start_datetime, and end_datetime fields.
 
         Args:
             search: The search object to filter.
-            interval: Optional datetime interval to filter by. Can be:
-                - A single datetime string (e.g., "2023-01-01T12:00:00")
-                - A datetime range string (e.g., "2023-01-01/2023-12-31")
-                - A datetime object
-                - A tuple of (start_datetime, end_datetime)
+            datetime: Optional[str]
 
         Returns:
             The filtered search object.
         """
-        if not interval:
-            return search
+        datetime_search = return_date(datetime)
 
-        should = []
-        try:
-            datetime_search = return_date(interval)
-        except (ValueError, TypeError) as e:
-            # Handle invalid interval formats if return_date fails
-            logger.error(f"Invalid interval format: {interval}, error: {e}")
-            return search
+        if not datetime_search:
+            return search, datetime_search
 
         if "eq" in datetime_search:
             # For exact matches, include:
@@ -387,7 +366,10 @@ class DatabaseLogic(BaseDatabaseLogic):
                 ),
             ]
 
-        return search.query(Q("bool", should=should, minimum_should_match=1))
+        return (
+            search.query(Q("bool", should=should, minimum_should_match=1)),
+            datetime_search,
+        )
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):
@@ -484,7 +466,7 @@ class DatabaseLogic(BaseDatabaseLogic):
                     otherwise the original Search object.
         """
         if _filter is not None:
-            es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
+            es_query = filter_module.to_es(await self.get_queryables_mapping(), _filter)
             search = search.filter(es_query)
 
         return search
@@ -511,6 +493,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         token: Optional[str],
         sort: Optional[Dict[str, Dict[str, str]]],
         collection_ids: Optional[List[str]],
+        datetime_search: Dict[str, Optional[str]],
         ignore_unavailable: bool = True,
     ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]:
         """Execute a search query with limit and other optional parameters.
@@ -521,6 +504,7 @@ class DatabaseLogic(BaseDatabaseLogic):
             token (Optional[str]): The token used to return the next set of results.
             sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted.
             collection_ids (Optional[List[str]]): The collection ids to search.
+            datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection.
             ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.
 
         Returns:
@@ -537,7 +521,9 @@ class DatabaseLogic(BaseDatabaseLogic):
         search_body: Dict[str, Any] = {}
         query = search.query.to_dict() if search.query else None
 
-        index_param = indices(collection_ids)
+        index_param = await self.async_index_selector.select_indexes(
+            collection_ids, datetime_search
+        )
         if len(index_param) > ES_MAX_URL_LENGTH - 300:
             index_param = ITEM_INDICES
             query = add_collections_to_body(collection_ids, query)
@@ -614,6 +600,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         geometry_geohash_grid_precision: int,
         geometry_geotile_grid_precision: int,
         datetime_frequency_interval: str,
+        datetime_search,
         ignore_unavailable: Optional[bool] = True,
     ):
         """Return aggregations of STAC Items."""
@@ -647,7 +634,10 @@ class DatabaseLogic(BaseDatabaseLogic):
             if k in aggregations
         }
 
-        index_param = indices(collection_ids)
+        index_param = await self.async_index_selector.select_indexes(
+            collection_ids, datetime_search
+        )
+
         search_task = asyncio.create_task(
             self.client.search(
                 index=index_param,
@@ -840,8 +830,13 @@ class DatabaseLogic(BaseDatabaseLogic):
         item = await self.async_prep_create_item(
             item=item, base_url=base_url, exist_ok=exist_ok
         )
+
+        target_index = await self.async_index_inserter.get_target_index(
+            collection_id, item
+        )
+
         await self.client.index(
-            index=index_alias_by_collection_id(collection_id),
+            index=target_index,
             id=mk_item_id(item_id, collection_id),
             body=item,
             refresh=refresh,
@@ -920,13 +915,28 @@ class DatabaseLogic(BaseDatabaseLogic):
         script = operations_to_script(script_operations)
 
         try:
-            await self.client.update(
+            search_response = await self.client.search(
                 index=index_alias_by_collection_id(collection_id),
+                body={
+                    "query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
+                    "size": 1,
+                },
+            )
+            if search_response["hits"]["total"]["value"] == 0:
+                raise NotFoundError(
+                    f"Item {item_id} does not exist inside Collection {collection_id}"
+                )
+            document_index = search_response["hits"]["hits"][0]["_index"]
+            await self.client.update(
+                index=document_index,
                 id=mk_item_id(item_id, collection_id),
                 body={"script": script},
                 refresh=True,
             )
-
+        except exceptions.NotFoundError:
+            raise NotFoundError(
+                f"Item {item_id} does not exist inside Collection {collection_id}"
+            )
         except exceptions.RequestError as exc:
             raise HTTPException(
                 status_code=400, detail=exc.info["error"]["caused_by"]
@@ -945,8 +955,8 @@ class DatabaseLogic(BaseDatabaseLogic):
                     "script": {
                         "lang": "painless",
                         "source": (
-                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""
-                            f"""ctx._source.collection = '{new_collection_id}';"""
+                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""  # noqa: E702
+                            f"""ctx._source.collection = '{new_collection_id}';"""  # noqa: E702
                         ),
                     },
                 },
@@ -1000,9 +1010,9 @@ class DatabaseLogic(BaseDatabaseLogic):
         )
 
         try:
-            await self.client.delete(
+            await self.client.delete_by_query(
                 index=index_alias_by_collection_id(collection_id),
-                id=mk_item_id(item_id, collection_id),
+                body={"query": {"term": {"_id": mk_item_id(item_id, collection_id)}}},
                 refresh=refresh,
             )
         except exceptions.NotFoundError:
@@ -1093,8 +1103,10 @@ class DatabaseLogic(BaseDatabaseLogic):
             body=collection,
             refresh=refresh,
         )
-
-        await create_item_index(collection_id)
+        if self.async_index_inserter.should_create_collection_index():
+            await self.async_index_inserter.create_simple_index(
+                self.client, collection_id
+            )
 
     async def find_collection(self, collection_id: str) -> Collection:
         """Find and return a collection from the database.
@@ -1303,6 +1315,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         await self.client.delete(
             index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh
         )
+        # Delete the item index for the collection
         await delete_item_index(collection_id)
 
     async def bulk_async(
@@ -1356,9 +1369,13 @@ class DatabaseLogic(BaseDatabaseLogic):
             return 0, []
 
         raise_on_error = self.async_settings.raise_on_bulk_error
+        actions = await self.async_index_inserter.prepare_bulk_actions(
+            collection_id, processed_items
+        )
+
         success, errors = await helpers.async_bulk(
             self.client,
-            mk_actions(collection_id, processed_items),
+            actions,
             refresh=refresh,
             raise_on_error=raise_on_error,
         )
@@ -1413,6 +1430,11 @@ class DatabaseLogic(BaseDatabaseLogic):
             f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
         )
 
+        # Handle empty processed_items
+        if not processed_items:
+            logger.warning(f"No items to insert for collection {collection_id}")
+            return 0, []
+
         # Handle empty processed_items
         if not processed_items:
             logger.warning(f"No items to insert for collection {collection_id}")

stac_fastapi/opensearch/version.py

@@ -1,2 +1,2 @@
 """library version."""
-__version__ = "6.1.0"
+__version__ = "6.2.0"

{stac_fastapi_opensearch-6.1.0.dist-info → stac_fastapi_opensearch-6.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stac-fastapi-opensearch
-Version: 6.1.0
+Version: 6.2.0
 Summary: Opensearch stac-fastapi backend.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT
@@ -15,8 +15,8 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: stac-fastapi-core==6.1.0
-Requires-Dist: sfeos-helpers==6.1.0
+Requires-Dist: stac-fastapi-core==6.2.0
+Requires-Dist: sfeos-helpers==6.2.0
 Requires-Dist: opensearch-py~=2.8.0
 Requires-Dist: opensearch-py[async]~=2.8.0
 Requires-Dist: uvicorn~=0.23.0
@@ -122,6 +122,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
 - [Auth](#auth)
 - [Aggregation](#aggregation)
 - [Rate Limiting](#rate-limiting)
+- [Datetime-Based Index Management](#datetime-based-index-management)
 
 ## Documentation & Resources
 
@@ -267,6 +268,81 @@ You can customize additional settings in your `.env` file:
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
 
+## Datetime-Based Index Management
+
+### Overview
+
+SFEOS supports two indexing strategies for managing STAC items:
+
+1. **Simple Indexing** (default) - One index per collection
+2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
+
+The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
+
+### When to Use
+
+**Recommended for:**
+- Systems with large collections containing millions of items
+- Systems requiring high-performance temporal searching
+
+**Pros:**
+- Multiple times faster queries with datetime filter
+- Reduced database load - only relevant indexes are searched
+
+**Cons:**
+- Slightly longer item indexing time (automatic index management)
+- Greater management complexity
+
+### Configuration
+
+#### Enabling Datetime-Based Indexing
+
+Enable datetime-based indexing by setting the following environment variable:
+
+```bash
+ENABLE_DATETIME_INDEX_FILTERING=true
+```
+
+### Related Configuration Variables
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
+| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
+| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
+
+## How Datetime-Based Indexing Works
+
+### Index and Alias Naming Convention
+
+The system uses a precise naming convention:
+
+**Physical indexes:**
+```
+{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
+```
+
+**Aliases:**
+```
+{ITEMS_INDEX_PREFIX}{collection-id}                                  # Main collection alias
+{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}                 # Temporal alias
+{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime}  # Closed index alias
+```
+
+**Example:**
+
+*Physical indexes:*
+- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
+
+*Aliases:*
+- `items_sentinel-2-l2a` - main collection alias
+- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
+- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
+
+### Index Size Management
+
+**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
+
 ## Interacting with the API
 
 - **Creating a Collection**:
@@ -575,4 +651,3 @@ You can customize additional settings in your `.env` file:
   - Ensures fair resource allocation among all clients
   
 - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
-

stac_fastapi_opensearch-6.2.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+stac_fastapi/opensearch/__init__.py,sha256=iJWMUgn7mUvmuPQSO_FlyhJ5eDdbbfmGv1qnFOX5-qk,28
+stac_fastapi/opensearch/app.py,sha256=WHo0C9c4EvNUW8l_2eiNu2yLPGe23RfHdu_F9-deoL0,5642
+stac_fastapi/opensearch/config.py,sha256=zGx4-4c5zEnu_Bh8Ra3SkIC83tluDiz-wKYQclRRDJA,5179
+stac_fastapi/opensearch/database_logic.py,sha256=0NzzWZqxrgZ-zcxqBcypOXeuFqf5EYia91za9NHepzY,55834
+stac_fastapi/opensearch/version.py,sha256=ro2d3oERQL2KxSo7qmbU0z6qT77XShwY6vsqrLf2VFw,45
+stac_fastapi_opensearch-6.2.0.dist-info/METADATA,sha256=0YnHUYtuSPuibokLp7ToJsEw4v9gIKbUoALnSchZuak,35019
+stac_fastapi_opensearch-6.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+stac_fastapi_opensearch-6.2.0.dist-info/entry_points.txt,sha256=zjZ0Xsr9BUNJqMkdPpl6zEIUykv1uFdJtNELFRChp0w,76
+stac_fastapi_opensearch-6.2.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
+stac_fastapi_opensearch-6.2.0.dist-info/RECORD,,

stac_fastapi_opensearch-6.1.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-stac_fastapi/opensearch/__init__.py,sha256=iJWMUgn7mUvmuPQSO_FlyhJ5eDdbbfmGv1qnFOX5-qk,28
-stac_fastapi/opensearch/app.py,sha256=EBaN0W8-jP9Q568J6UlC_xM7uWx7PkfE4RulnxYJCYs,5642
-stac_fastapi/opensearch/config.py,sha256=tR-CP3l96pte0gdbQqDHAQVZrWbL57krMrFalLKCTBc,5178
-stac_fastapi/opensearch/database_logic.py,sha256=9c2UKJcFaaZ9fcXUkCYnDy06G16BHGu96kb13Clg0ow,54664
-stac_fastapi/opensearch/version.py,sha256=7IrY7mbr0cGVqZsk6wmCeITxZjDgz_mPHUswrziX5ME,45
-stac_fastapi_opensearch-6.1.0.dist-info/METADATA,sha256=CgFBwwx65wUV-jcw3sbSFhUKcre3GgfWUBlwEhOQRuM,32250
-stac_fastapi_opensearch-6.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-stac_fastapi_opensearch-6.1.0.dist-info/entry_points.txt,sha256=zjZ0Xsr9BUNJqMkdPpl6zEIUykv1uFdJtNELFRChp0w,76
-stac_fastapi_opensearch-6.1.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
-stac_fastapi_opensearch-6.1.0.dist-info/RECORD,,