stac-fastapi-elasticsearch 6.1.0__py3-none-any.whl → 6.2.0__py3-none-any.whl

stac_fastapi/elasticsearch/app.py

@@ -117,7 +117,7 @@ post_request_model = create_post_request_model(search_extensions)
 app_config = {
     "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"),
     "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"),
-    "api_version": os.getenv("STAC_FASTAPI_VERSION", "6.1.0"),
+    "api_version": os.getenv("STAC_FASTAPI_VERSION", "6.2.0"),
     "settings": settings,
     "extensions": extensions,
     "client": CoreClient(

stac_fastapi/elasticsearch/database_logic.py

@@ -4,7 +4,7 @@ import asyncio
 import logging
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
 
 import attr
 import elasticsearch.helpers as helpers
@@ -27,7 +27,7 @@ from stac_fastapi.extensions.core.transaction.request import (
     PartialItem,
     PatchOperation,
 )
-from stac_fastapi.sfeos_helpers import filter
+from stac_fastapi.sfeos_helpers import filter as filter_module
 from stac_fastapi.sfeos_helpers.database import (
     apply_free_text_filter_shared,
     apply_intersects_filter_shared,
@@ -36,7 +36,6 @@ from stac_fastapi.sfeos_helpers.database import (
     get_queryables_mapping_shared,
     index_alias_by_collection_id,
     index_by_collection_id,
-    indices,
     mk_actions,
     mk_item_id,
     populate_sort_shared,
@@ -59,9 +58,14 @@ from stac_fastapi.sfeos_helpers.mappings import (
     ITEMS_INDEX_PREFIX,
     Geometry,
 )
+from stac_fastapi.sfeos_helpers.search_engine import (
+    BaseIndexInserter,
+    BaseIndexSelector,
+    IndexInsertionFactory,
+    IndexSelectorFactory,
+)
 from stac_fastapi.types.errors import ConflictError, NotFoundError
 from stac_fastapi.types.links import resolve_links
-from stac_fastapi.types.rfc3339 import DateTimeType
 from stac_fastapi.types.stac import Collection, Item
 
 logger = logging.getLogger(__name__)
@@ -139,6 +143,8 @@ class DatabaseLogic(BaseDatabaseLogic):
     sync_settings: SyncElasticsearchSettings = attr.ib(
         factory=SyncElasticsearchSettings
     )
+    async_index_selector: BaseIndexSelector = attr.ib(init=False)
+    async_index_inserter: BaseIndexInserter = attr.ib(init=False)
 
     client = attr.ib(init=False)
     sync_client = attr.ib(init=False)
@@ -147,6 +153,10 @@ class DatabaseLogic(BaseDatabaseLogic):
         """Initialize clients after the class is instantiated."""
         self.client = self.async_settings.create_client
         self.sync_client = self.sync_settings.create_client
+        self.async_index_inserter = IndexInsertionFactory.create_insertion_strategy(
+            self.client
+        )
+        self.async_index_selector = IndexSelectorFactory.create_selector(self.client)
 
     item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
     collection_serializer: Type[CollectionSerializer] = attr.ib(
@@ -216,15 +226,23 @@ class DatabaseLogic(BaseDatabaseLogic):
             with the index for the Collection as the target index and the combined `mk_item_id` as the document id.
         """
         try:
-            item = await self.client.get(
+            response = await self.client.search(
                 index=index_alias_by_collection_id(collection_id),
-                id=mk_item_id(item_id, collection_id),
+                body={
+                    "query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
+                    "size": 1,
+                },
             )
+            if response["hits"]["total"]["value"] == 0:
+                raise NotFoundError(
+                    f"Item {item_id} does not exist inside Collection {collection_id}"
+                )
+
+            return response["hits"]["hits"][0]["_source"]
         except ESNotFoundError:
             raise NotFoundError(
                 f"Item {item_id} does not exist inside Collection {collection_id}"
             )
-        return item["_source"]
 
     async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
         """Retrieve mapping of Queryables for search.
@@ -260,31 +278,21 @@ class DatabaseLogic(BaseDatabaseLogic):
 
     @staticmethod
     def apply_datetime_filter(
-        search: Search, interval: Optional[Union[DateTimeType, str]]
-    ) -> Search:
+        search: Search, datetime: Optional[str]
+    ) -> Tuple[Search, Dict[str, Optional[str]]]:
         """Apply a filter to search on datetime, start_datetime, and end_datetime fields.
 
         Args:
             search: The search object to filter.
-            interval: Optional datetime interval to filter by. Can be:
-                - A single datetime string (e.g., "2023-01-01T12:00:00")
-                - A datetime range string (e.g., "2023-01-01/2023-12-31")
-                - A datetime object
-                - A tuple of (start_datetime, end_datetime)
+            datetime: Optional[str]
 
         Returns:
             The filtered search object.
         """
-        if not interval:
-            return search
+        datetime_search = return_date(datetime)
 
-        should = []
-        try:
-            datetime_search = return_date(interval)
-        except (ValueError, TypeError) as e:
-            # Handle invalid interval formats if return_date fails
-            logger.error(f"Invalid interval format: {interval}, error: {e}")
-            return search
+        if not datetime_search:
+            return search, datetime_search
 
         if "eq" in datetime_search:
             # For exact matches, include:
@@ -351,7 +359,10 @@ class DatabaseLogic(BaseDatabaseLogic):
                 ),
             ]
 
-        return search.query(Q("bool", should=should, minimum_should_match=1))
+        return (
+            search.query(Q("bool", should=should, minimum_should_match=1)),
+            datetime_search,
+        )
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):
@@ -466,7 +477,7 @@ class DatabaseLogic(BaseDatabaseLogic):
                     otherwise the original Search object.
         """
         if _filter is not None:
-            es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
+            es_query = filter_module.to_es(await self.get_queryables_mapping(), _filter)
             search = search.query(es_query)
 
         return search
@@ -493,6 +504,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         token: Optional[str],
         sort: Optional[Dict[str, Dict[str, str]]],
         collection_ids: Optional[List[str]],
+        datetime_search: Dict[str, Optional[str]],
         ignore_unavailable: bool = True,
     ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]:
         """Execute a search query with limit and other optional parameters.
@@ -503,6 +515,7 @@ class DatabaseLogic(BaseDatabaseLogic):
             token (Optional[str]): The token used to return the next set of results.
             sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted.
             collection_ids (Optional[List[str]]): The collection ids to search.
+            datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection.
             ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.
 
         Returns:
@@ -523,7 +536,9 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         query = search.query.to_dict() if search.query else None
 
-        index_param = indices(collection_ids)
+        index_param = await self.async_index_selector.select_indexes(
+            collection_ids, datetime_search
+        )
         if len(index_param) > ES_MAX_URL_LENGTH - 300:
             index_param = ITEM_INDICES
             query = add_collections_to_body(collection_ids, query)
@@ -590,6 +605,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         geometry_geohash_grid_precision: int,
         geometry_geotile_grid_precision: int,
         datetime_frequency_interval: str,
+        datetime_search,
         ignore_unavailable: Optional[bool] = True,
     ):
         """Return aggregations of STAC Items."""
@@ -625,7 +641,10 @@ class DatabaseLogic(BaseDatabaseLogic):
             if k in aggregations
         }
 
-        index_param = indices(collection_ids)
+        index_param = await self.async_index_selector.select_indexes(
+            collection_ids, datetime_search
+        )
+
         search_task = asyncio.create_task(
             self.client.search(
                 index=index_param,
@@ -667,14 +686,21 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         """
         await self.check_collection_exists(collection_id=item["collection"])
+        alias = index_alias_by_collection_id(item["collection"])
+        doc_id = mk_item_id(item["id"], item["collection"])
 
-        if not exist_ok and await self.client.exists(
-            index=index_alias_by_collection_id(item["collection"]),
-            id=mk_item_id(item["id"], item["collection"]),
-        ):
-            raise ConflictError(
-                f"Item {item['id']} in collection {item['collection']} already exists"
-            )
+        if not exist_ok:
+            alias_exists = await self.client.indices.exists_alias(name=alias)
+
+            if alias_exists:
+                alias_info = await self.client.indices.get_alias(name=alias)
+                indices = list(alias_info.keys())
+
+                for index in indices:
+                    if await self.client.exists(index=index, id=doc_id):
+                        raise ConflictError(
+                            f"Item {item['id']} in collection {item['collection']} already exists"
+                        )
 
         return self.item_serializer.stac_to_db(item, base_url)
 
@@ -805,7 +831,6 @@ class DatabaseLogic(BaseDatabaseLogic):
         # Extract item and collection IDs
         item_id = item["id"]
         collection_id = item["collection"]
-
         # Ensure kwargs is a dictionary
         kwargs = kwargs or {}
 
@@ -823,9 +848,12 @@ class DatabaseLogic(BaseDatabaseLogic):
             item=item, base_url=base_url, exist_ok=exist_ok
         )
 
+        target_index = await self.async_index_inserter.get_target_index(
+            collection_id, item
+        )
         # Index the item in the database
         await self.client.index(
-            index=index_alias_by_collection_id(collection_id),
+            index=target_index,
             id=mk_item_id(item_id, collection_id),
             document=item,
             refresh=refresh,
@@ -904,13 +932,28 @@ class DatabaseLogic(BaseDatabaseLogic):
         script = operations_to_script(script_operations)
 
         try:
-            await self.client.update(
+            search_response = await self.client.search(
                 index=index_alias_by_collection_id(collection_id),
+                body={
+                    "query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
+                    "size": 1,
+                },
+            )
+            if search_response["hits"]["total"]["value"] == 0:
+                raise NotFoundError(
+                    f"Item {item_id} does not exist inside Collection {collection_id}"
+                )
+            document_index = search_response["hits"]["hits"][0]["_index"]
+            await self.client.update(
+                index=document_index,
                 id=mk_item_id(item_id, collection_id),
                 script=script,
                 refresh=True,
             )
-
+        except ESNotFoundError:
+            raise NotFoundError(
+                f"Item {item_id} does not exist inside Collection {collection_id}"
+            )
         except BadRequestError as exc:
             raise HTTPException(
                 status_code=400, detail=exc.info["error"]["caused_by"]
@@ -921,7 +964,9 @@ class DatabaseLogic(BaseDatabaseLogic):
         if new_collection_id:
             await self.client.reindex(
                 body={
-                    "dest": {"index": f"{ITEMS_INDEX_PREFIX}{new_collection_id}"},
+                    "dest": {
+                        "index": f"{ITEMS_INDEX_PREFIX}{new_collection_id}"
+                    },  # # noqa
                     "source": {
                         "index": f"{ITEMS_INDEX_PREFIX}{collection_id}",
                         "query": {"term": {"id": {"value": item_id}}},
@@ -929,8 +974,8 @@ class DatabaseLogic(BaseDatabaseLogic):
                     "script": {
                         "lang": "painless",
                         "source": (
-                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""
-                            f"""ctx._source.collection = '{new_collection_id}';"""
+                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""  # noqa
+                            f"""ctx._source.collection = '{new_collection_id}';"""  # noqa
                         ),
                     },
                 },
@@ -990,9 +1035,9 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         try:
             # Perform the delete operation
-            await self.client.delete(
+            await self.client.delete_by_query(
                 index=index_alias_by_collection_id(collection_id),
-                id=mk_item_id(item_id, collection_id),
+                body={"query": {"term": {"_id": mk_item_id(item_id, collection_id)}}},
                 refresh=refresh,
             )
         except ESNotFoundError:
@@ -1092,8 +1137,10 @@ class DatabaseLogic(BaseDatabaseLogic):
             refresh=refresh,
         )
 
-        # Create the item index for the collection
-        await create_item_index(collection_id)
+        if self.async_index_inserter.should_create_collection_index():
+            await self.async_index_inserter.create_simple_index(
+                self.client, collection_id
+            )
 
     async def find_collection(self, collection_id: str) -> Collection:
         """Find and return a collection from the database.
@@ -1367,9 +1414,12 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         # Perform the bulk insert
         raise_on_error = self.async_settings.raise_on_bulk_error
+        actions = await self.async_index_inserter.prepare_bulk_actions(
+            collection_id, processed_items
+        )
         success, errors = await helpers.async_bulk(
             self.client,
-            mk_actions(collection_id, processed_items),
+            actions,
             refresh=refresh,
             raise_on_error=raise_on_error,
         )

stac_fastapi/elasticsearch/version.py

@@ -1,2 +1,2 @@
 """library version."""
-__version__ = "6.1.0"
+__version__ = "6.2.0"

{stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stac-fastapi-elasticsearch
-Version: 6.1.0
+Version: 6.2.0
 Summary: An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT
@@ -15,8 +15,8 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: stac-fastapi-core==6.1.0
-Requires-Dist: sfeos-helpers==6.1.0
+Requires-Dist: stac-fastapi-core==6.2.0
+Requires-Dist: sfeos-helpers==6.2.0
 Requires-Dist: elasticsearch[async]~=8.18.0
 Requires-Dist: uvicorn~=0.23.0
 Requires-Dist: starlette<0.36.0,>=0.35.0
@@ -121,6 +121,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
 - [Auth](#auth)
 - [Aggregation](#aggregation)
 - [Rate Limiting](#rate-limiting)
+- [Datetime-Based Index Management](#datetime-based-index-management)
 
 ## Documentation & Resources
 
@@ -266,6 +267,81 @@ You can customize additional settings in your `.env` file:
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
 
+## Datetime-Based Index Management
+
+### Overview
+
+SFEOS supports two indexing strategies for managing STAC items:
+
+1. **Simple Indexing** (default) - One index per collection
+2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
+
+The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
+
+### When to Use
+
+**Recommended for:**
+- Systems with large collections containing millions of items
+- Systems requiring high-performance temporal searching
+
+**Pros:**
+- Multiple times faster queries with datetime filter
+- Reduced database load - only relevant indexes are searched
+
+**Cons:**
+- Slightly longer item indexing time (automatic index management)
+- Greater management complexity
+
+### Configuration
+
+#### Enabling Datetime-Based Indexing
+
+Enable datetime-based indexing by setting the following environment variable:
+
+```bash
+ENABLE_DATETIME_INDEX_FILTERING=true
+```
+
+### Related Configuration Variables
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
+| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
+| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
+
+## How Datetime-Based Indexing Works
+
+### Index and Alias Naming Convention
+
+The system uses a precise naming convention:
+
+**Physical indexes:**
+```
+{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
+```
+
+**Aliases:**
+```
+{ITEMS_INDEX_PREFIX}{collection-id}                                  # Main collection alias
+{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}                 # Temporal alias
+{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime}  # Closed index alias
+```
+
+**Example:**
+
+*Physical indexes:*
+- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
+
+*Aliases:*
+- `items_sentinel-2-l2a` - main collection alias
+- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
+- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
+
+### Index Size Management
+
+**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
+
 ## Interacting with the API
 
 - **Creating a Collection**:
@@ -574,4 +650,3 @@ You can customize additional settings in your `.env` file:
   - Ensures fair resource allocation among all clients
   
 - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
-

stac_fastapi_elasticsearch-6.2.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
+stac_fastapi/elasticsearch/app.py,sha256=Sdtyk4yeHJtbDim1rwLcKyDVLRQCSdAQV7JbD-fDMnk,5662
+stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
+stac_fastapi/elasticsearch/database_logic.py,sha256=kG_GBGJxQlLDs-CRT9Jysesrrn-uKY2r6tS1rB_he30,58298
+stac_fastapi/elasticsearch/version.py,sha256=ro2d3oERQL2KxSo7qmbU0z6qT77XShwY6vsqrLf2VFw,45
+stac_fastapi_elasticsearch-6.2.0.dist-info/METADATA,sha256=uiqM95BihA3TWSwzLb9hRPbCec-SmnM8o-Vz8rgqqdI,35055
+stac_fastapi_elasticsearch-6.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+stac_fastapi_elasticsearch-6.2.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
+stac_fastapi_elasticsearch-6.2.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
+stac_fastapi_elasticsearch-6.2.0.dist-info/RECORD,,

stac_fastapi_elasticsearch-6.1.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
-stac_fastapi/elasticsearch/app.py,sha256=EK0H5qiRX-frbSHkBkZ_4Lzo0Jn8hBzJckSLCYbQlKs,5662
-stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
-stac_fastapi/elasticsearch/database_logic.py,sha256=8P-eUGkDQAABOJWaYd_NTHpKpKug7o2c6BMDWYHH5m4,56201
-stac_fastapi/elasticsearch/version.py,sha256=7IrY7mbr0cGVqZsk6wmCeITxZjDgz_mPHUswrziX5ME,45
-stac_fastapi_elasticsearch-6.1.0.dist-info/METADATA,sha256=a2Z6VUi-YBORUsPZZDBJt1HuLqPeEPKtFgE2PqKEXj8,32286
-stac_fastapi_elasticsearch-6.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-stac_fastapi_elasticsearch-6.1.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
-stac_fastapi_elasticsearch-6.1.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
-stac_fastapi_elasticsearch-6.1.0.dist-info/RECORD,,