stac-fastapi-elasticsearch 4.1.0__py3-none-any.whl → 5.0.0__py3-none-any.whl

stac_fastapi/elasticsearch/database_logic.py

@@ -1,42 +1,51 @@
 """Database logic."""
 
 import asyncio
-import json
 import logging
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
 
 import attr
 import elasticsearch.helpers as helpers
+import orjson
 from elasticsearch.dsl import Q, Search
 from elasticsearch.exceptions import NotFoundError as ESNotFoundError
 from starlette.requests import Request
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.database_logic import (
-    COLLECTIONS_INDEX,
-    DEFAULT_SORT,
-    ES_COLLECTIONS_MAPPINGS,
-    ES_ITEMS_MAPPINGS,
-    ES_ITEMS_SETTINGS,
-    ITEM_INDICES,
-    ITEMS_INDEX_PREFIX,
-    Geometry,
+from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
+from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
+from stac_fastapi.elasticsearch.config import (
+    ElasticsearchSettings as SyncElasticsearchSettings,
+)
+from stac_fastapi.sfeos_helpers import filter
+from stac_fastapi.sfeos_helpers.database import (
+    apply_free_text_filter_shared,
+    apply_intersects_filter_shared,
+    create_index_templates_shared,
+    delete_item_index_shared,
+    get_queryables_mapping_shared,
     index_alias_by_collection_id,
     index_by_collection_id,
     indices,
     mk_actions,
     mk_item_id,
+    populate_sort_shared,
+    return_date,
+    validate_refresh,
 )
-from stac_fastapi.core.extensions import filter
-from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
-from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
-from stac_fastapi.elasticsearch.config import (
-    ElasticsearchSettings as SyncElasticsearchSettings,
+from stac_fastapi.sfeos_helpers.mappings import (
+    AGGREGATION_MAPPING,
+    COLLECTIONS_INDEX,
+    DEFAULT_SORT,
+    ITEM_INDICES,
+    ITEMS_INDEX_PREFIX,
+    Geometry,
 )
 from stac_fastapi.types.errors import ConflictError, NotFoundError
+from stac_fastapi.types.rfc3339 import DateTimeType
 from stac_fastapi.types.stac import Collection, Item
 
 logger = logging.getLogger(__name__)
@@ -50,22 +59,7 @@ async def create_index_templates() -> None:
         None
 
     """
-    client = AsyncElasticsearchSettings().create_client
-    await client.indices.put_index_template(
-        name=f"template_{COLLECTIONS_INDEX}",
-        body={
-            "index_patterns": [f"{COLLECTIONS_INDEX}*"],
-            "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
-        },
-    )
-    await client.indices.put_index_template(
-        name=f"template_{ITEMS_INDEX_PREFIX}",
-        body={
-            "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
-            "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
-        },
-    )
-    await client.close()
+    await create_index_templates_shared(settings=AsyncElasticsearchSettings())
 
 
 async def create_collection_index() -> None:
@@ -110,18 +104,13 @@ async def delete_item_index(collection_id: str):
 
     Args:
         collection_id (str): The ID of the collection whose items index will be deleted.
-    """
-    client = AsyncElasticsearchSettings().create_client
 
-    name = index_alias_by_collection_id(collection_id)
-    resolved = await client.indices.resolve_index(name=name)
-    if "aliases" in resolved and resolved["aliases"]:
-        [alias] = resolved["aliases"]
-        await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
-        await client.indices.delete(index=alias["indices"])
-    else:
-        await client.indices.delete(index=name)
-    await client.close()
+    Notes:
+        This function delegates to the shared implementation in delete_item_index_shared.
+    """
+    await delete_item_index_shared(
+        settings=AsyncElasticsearchSettings(), collection_id=collection_id
+    )
 
 
 @attr.s
@@ -150,76 +139,7 @@ class DatabaseLogic(BaseDatabaseLogic):
 
     extensions: List[str] = attr.ib(default=attr.Factory(list))
 
-    aggregation_mapping: Dict[str, Dict[str, Any]] = {
-        "total_count": {"value_count": {"field": "id"}},
-        "collection_frequency": {"terms": {"field": "collection", "size": 100}},
-        "platform_frequency": {"terms": {"field": "properties.platform", "size": 100}},
-        "cloud_cover_frequency": {
-            "range": {
-                "field": "properties.eo:cloud_cover",
-                "ranges": [
-                    {"to": 5},
-                    {"from": 5, "to": 15},
-                    {"from": 15, "to": 40},
-                    {"from": 40},
-                ],
-            }
-        },
-        "datetime_frequency": {
-            "date_histogram": {
-                "field": "properties.datetime",
-                "calendar_interval": "month",
-            }
-        },
-        "datetime_min": {"min": {"field": "properties.datetime"}},
-        "datetime_max": {"max": {"field": "properties.datetime"}},
-        "grid_code_frequency": {
-            "terms": {
-                "field": "properties.grid:code",
-                "missing": "none",
-                "size": 10000,
-            }
-        },
-        "sun_elevation_frequency": {
-            "histogram": {"field": "properties.view:sun_elevation", "interval": 5}
-        },
-        "sun_azimuth_frequency": {
-            "histogram": {"field": "properties.view:sun_azimuth", "interval": 5}
-        },
-        "off_nadir_frequency": {
-            "histogram": {"field": "properties.view:off_nadir", "interval": 5}
-        },
-        "centroid_geohash_grid_frequency": {
-            "geohash_grid": {
-                "field": "properties.proj:centroid",
-                "precision": 1,
-            }
-        },
-        "centroid_geohex_grid_frequency": {
-            "geohex_grid": {
-                "field": "properties.proj:centroid",
-                "precision": 0,
-            }
-        },
-        "centroid_geotile_grid_frequency": {
-            "geotile_grid": {
-                "field": "properties.proj:centroid",
-                "precision": 0,
-            }
-        },
-        "geometry_geohash_grid_frequency": {
-            "geohash_grid": {
-                "field": "geometry",
-                "precision": 1,
-            }
-        },
-        "geometry_geotile_grid_frequency": {
-            "geotile_grid": {
-                "field": "geometry",
-                "precision": 0,
-            }
-        },
-    }
+    aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING
 
     """CORE LOGIC"""
 
@@ -290,6 +210,23 @@ class DatabaseLogic(BaseDatabaseLogic):
             )
         return item["_source"]
 
+    async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
+        """Retrieve mapping of Queryables for search.
+
+        Args:
+            collection_id (str, optional): The id of the Collection the Queryables
+            belongs to. Defaults to "*".
+
+        Returns:
+            dict: A dictionary containing the Queryables mappings.
+        """
+        mappings = await self.client.indices.get_mapping(
+            index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
+        )
+        return await get_queryables_mapping_shared(
+            collection_id=collection_id, mappings=mappings
+        )
+
     @staticmethod
     def make_search():
         """Database logic to create a Search instance."""
@@ -306,120 +243,99 @@ class DatabaseLogic(BaseDatabaseLogic):
         return search.filter("terms", collection=collection_ids)
 
     @staticmethod
-    def apply_datetime_filter(search: Search, datetime_search: dict):
+    def apply_datetime_filter(
+        search: Search, interval: Optional[Union[DateTimeType, str]]
+    ) -> Search:
         """Apply a filter to search on datetime, start_datetime, and end_datetime fields.
 
         Args:
-            search (Search): The search object to filter.
-            datetime_search (dict): The datetime filter criteria.
+            search: The search object to filter.
+            interval: Optional datetime interval to filter by. Can be:
+                - A single datetime string (e.g., "2023-01-01T12:00:00")
+                - A datetime range string (e.g., "2023-01-01/2023-12-31")
+                - A datetime object
+                - A tuple of (start_datetime, end_datetime)
 
         Returns:
-            Search: The filtered search object.
+            The filtered search object.
         """
+        if not interval:
+            return search
+
         should = []
+        try:
+            datetime_search = return_date(interval)
+        except (ValueError, TypeError) as e:
+            # Handle invalid interval formats if return_date fails
+            logger.error(f"Invalid interval format: {interval}, error: {e}")
+            return search
 
-        # If the request is a single datetime return
-        # items with datetimes equal to the requested datetime OR
-        # the requested datetime is between their start and end datetimes
         if "eq" in datetime_search:
-            should.extend(
-                [
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "term",
-                                properties__datetime=datetime_search["eq"],
-                            ),
-                        ],
-                    ),
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "range",
-                                properties__start_datetime={
-                                    "lte": datetime_search["eq"],
-                                },
-                            ),
-                            Q(
-                                "range",
-                                properties__end_datetime={
-                                    "gte": datetime_search["eq"],
-                                },
-                            ),
-                        ],
-                    ),
-                ]
-            )
-
-        # If the request is a date range return
-        # items with datetimes within the requested date range OR
-        # their startdatetime ithin the requested date range OR
-        # their enddatetime ithin the requested date range OR
-        # the requested daterange within their start and end datetimes
+            # For exact matches, include:
+            # 1. Items with matching exact datetime
+            # 2. Items with datetime:null where the time falls within their range
+            should = [
+                Q(
+                    "bool",
+                    filter=[
+                        Q("exists", field="properties.datetime"),
+                        Q("term", **{"properties__datetime": datetime_search["eq"]}),
+                    ],
+                ),
+                Q(
+                    "bool",
+                    must_not=[Q("exists", field="properties.datetime")],
+                    filter=[
+                        Q("exists", field="properties.start_datetime"),
+                        Q("exists", field="properties.end_datetime"),
+                        Q(
+                            "range",
+                            properties__start_datetime={"lte": datetime_search["eq"]},
+                        ),
+                        Q(
+                            "range",
+                            properties__end_datetime={"gte": datetime_search["eq"]},
+                        ),
+                    ],
+                ),
+            ]
         else:
-            should.extend(
-                [
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "range",
-                                properties__datetime={
-                                    "gte": datetime_search["gte"],
-                                    "lte": datetime_search["lte"],
-                                },
-                            ),
-                        ],
-                    ),
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "range",
-                                properties__start_datetime={
-                                    "gte": datetime_search["gte"],
-                                    "lte": datetime_search["lte"],
-                                },
-                            ),
-                        ],
-                    ),
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "range",
-                                properties__end_datetime={
-                                    "gte": datetime_search["gte"],
-                                    "lte": datetime_search["lte"],
-                                },
-                            ),
-                        ],
-                    ),
-                    Q(
-                        "bool",
-                        filter=[
-                            Q(
-                                "range",
-                                properties__start_datetime={
-                                    "lte": datetime_search["gte"]
-                                },
-                            ),
-                            Q(
-                                "range",
-                                properties__end_datetime={
-                                    "gte": datetime_search["lte"]
-                                },
-                            ),
-                        ],
-                    ),
-                ]
-            )
-
-        search = search.query(Q("bool", filter=[Q("bool", should=should)]))
-
-        return search
+            # For date ranges, include:
+            # 1. Items with datetime in the range
+            # 2. Items with datetime:null that overlap the search range
+            should = [
+                Q(
+                    "bool",
+                    filter=[
+                        Q("exists", field="properties.datetime"),
+                        Q(
+                            "range",
+                            properties__datetime={
+                                "gte": datetime_search["gte"],
+                                "lte": datetime_search["lte"],
+                            },
+                        ),
+                    ],
+                ),
+                Q(
+                    "bool",
+                    must_not=[Q("exists", field="properties.datetime")],
+                    filter=[
+                        Q("exists", field="properties.start_datetime"),
+                        Q("exists", field="properties.end_datetime"),
+                        Q(
+                            "range",
+                            properties__start_datetime={"lte": datetime_search["lte"]},
+                        ),
+                        Q(
+                            "range",
+                            properties__end_datetime={"gte": datetime_search["gte"]},
+                        ),
+                    ],
+                ),
+            ]
+
+        return search.query(Q("bool", should=should, minimum_should_match=1))
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):
@@ -469,21 +385,8 @@ class DatabaseLogic(BaseDatabaseLogic):
         Notes:
             A geo_shape filter is added to the search object, set to intersect with the specified geometry.
         """
-        return search.filter(
-            Q(
-                {
-                    "geo_shape": {
-                        "geometry": {
-                            "shape": {
-                                "type": intersects.type.lower(),
-                                "coordinates": intersects.coordinates,
-                            },
-                            "relation": "intersects",
-                        }
-                    }
-                }
-            )
-        )
+        filter = apply_intersects_filter_shared(intersects=intersects)
+        return search.filter(Q(filter))
 
     @staticmethod
     def apply_stacql_filter(search: Search, op: str, field: str, value: float):
@@ -509,17 +412,25 @@ class DatabaseLogic(BaseDatabaseLogic):
 
     @staticmethod
     def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]):
-        """Database logic to perform query for search endpoint."""
-        if free_text_queries is not None:
-            free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
-            search = search.query(
-                "query_string", query=f'properties.\\*:"{free_text_query_string}"'
-            )
+        """Create a free text query for Elasticsearch queries.
 
-        return search
+        This method delegates to the shared implementation in apply_free_text_filter_shared.
 
-    @staticmethod
-    def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]):
+        Args:
+            search (Search): The search object to apply the query to.
+            free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+        Returns:
+            Search: The search object with the free text query applied, or the original search
+                object if no free_text_queries were provided.
+        """
+        return apply_free_text_filter_shared(
+            search=search, free_text_queries=free_text_queries
+        )
+
+    async def apply_cql2_filter(
+        self, search: Search, _filter: Optional[Dict[str, Any]]
+    ):
         """
         Apply a CQL2 filter to an Elasticsearch Search object.
 
@@ -539,18 +450,25 @@ class DatabaseLogic(BaseDatabaseLogic):
                     otherwise the original Search object.
         """
         if _filter is not None:
-            es_query = filter.to_es(_filter)
+            es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
             search = search.query(es_query)
 
         return search
 
     @staticmethod
     def populate_sort(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
-        """Database logic to sort search instance."""
-        if sortby:
-            return {s.field: {"order": s.direction} for s in sortby}
-        else:
-            return None
+        """Create a sort configuration for Elasticsearch queries.
+
+        This method delegates to the shared implementation in populate_sort_shared.
+
+        Args:
+            sortby (List): A list of sort specifications, each containing a field and direction.
+
+        Returns:
+            Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+                configurations, or None if no sort was specified.
+        """
+        return populate_sort_shared(sortby=sortby)
 
     async def execute_search(
         self,
@@ -585,7 +503,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         search_after = None
 
         if token:
-            search_after = json.loads(urlsafe_b64decode(token).decode())
+            search_after = orjson.loads(urlsafe_b64decode(token))
 
         query = search.query.to_dict() if search.query else None
 
@@ -625,7 +543,7 @@ class DatabaseLogic(BaseDatabaseLogic):
         next_token = None
         if len(hits) > limit and limit < max_result_window:
             if hits and (sort_array := hits[limit - 1].get("sort")):
-                next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode()
+                next_token = urlsafe_b64encode(orjson.dumps(sort_array)).decode()
 
         matched = (
             es_response["hits"]["total"]["value"]
@@ -845,15 +763,19 @@ class DatabaseLogic(BaseDatabaseLogic):
     async def create_item(
         self,
         item: Item,
-        refresh: bool = False,
         base_url: str = "",
         exist_ok: bool = False,
+        **kwargs: Any,
     ):
         """Database logic for creating one item.
 
         Args:
             item (Item): The item to be created.
-            refresh (bool, optional): Refresh the index after performing the operation. Defaults to False.
+            base_url (str, optional): The base URL for the item. Defaults to an empty string.
+            exist_ok (bool, optional): Whether to allow the item to exist already. Defaults to False.
+            **kwargs: Additional keyword arguments.
+                - refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
+                - refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
 
         Raises:
             ConflictError: If the item already exists in the database.
@@ -861,12 +783,28 @@ class DatabaseLogic(BaseDatabaseLogic):
         Returns:
             None
         """
-        # todo: check if collection exists, but cache
+        # Extract item and collection IDs
         item_id = item["id"]
         collection_id = item["collection"]
+
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the creation attempt
+        logger.info(
+            f"Creating item {item_id} in collection {collection_id} with refresh={refresh}"
+        )
+
+        # Prepare the item for insertion
         item = await self.async_prep_create_item(
             item=item, base_url=base_url, exist_ok=exist_ok
         )
+
+        # Index the item in the database
         await self.client.index(
             index=index_alias_by_collection_id(collection_id),
             id=mk_item_id(item_id, collection_id),
@@ -874,26 +812,43 @@ class DatabaseLogic(BaseDatabaseLogic):
             refresh=refresh,
         )
 
-    async def delete_item(
-        self, item_id: str, collection_id: str, refresh: bool = False
-    ):
+    async def delete_item(self, item_id: str, collection_id: str, **kwargs: Any):
         """Delete a single item from the database.
 
         Args:
             item_id (str): The id of the Item to be deleted.
             collection_id (str): The id of the Collection that the Item belongs to.
-            refresh (bool, optional): Whether to refresh the index after the deletion. Default is False.
+            **kwargs: Additional keyword arguments.
+                - refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
+                - refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
 
         Raises:
             NotFoundError: If the Item does not exist in the database.
+
+        Returns:
+            None
         """
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the deletion attempt
+        logger.info(
+            f"Deleting item {item_id} from collection {collection_id} with refresh={refresh}"
+        )
+
         try:
+            # Perform the delete operation
             await self.client.delete(
                 index=index_alias_by_collection_id(collection_id),
                 id=mk_item_id(item_id, collection_id),
                 refresh=refresh,
             )
         except ESNotFoundError:
+            # Raise a custom NotFoundError if the item does not exist
             raise NotFoundError(
                 f"Item {item_id} in collection {collection_id} not found"
             )
@@ -916,24 +871,72 @@ class DatabaseLogic(BaseDatabaseLogic):
         except ESNotFoundError:
             raise NotFoundError(f"Mapping for index {index_name} not found")
 
-    async def create_collection(self, collection: Collection, refresh: bool = False):
+    async def get_items_unique_values(
+        self, collection_id: str, field_names: Iterable[str], *, limit: int = 100
+    ) -> Dict[str, List[str]]:
+        """Get the unique values for the given fields in the collection."""
+        limit_plus_one = limit + 1
+        index_name = index_alias_by_collection_id(collection_id)
+
+        query = await self.client.search(
+            index=index_name,
+            body={
+                "size": 0,
+                "aggs": {
+                    field: {"terms": {"field": field, "size": limit_plus_one}}
+                    for field in field_names
+                },
+            },
+        )
+
+        result: Dict[str, List[str]] = {}
+        for field, agg in query["aggregations"].items():
+            if len(agg["buckets"]) > limit:
+                logger.warning(
+                    "Skipping enum field %s: exceeds limit of %d unique values. "
+                    "Consider excluding this field from enumeration or increase the limit.",
+                    field,
+                    limit,
+                )
+                continue
+            result[field] = [bucket["key"] for bucket in agg["buckets"]]
+        return result
+
+    async def create_collection(self, collection: Collection, **kwargs: Any):
         """Create a single collection in the database.
 
         Args:
             collection (Collection): The Collection object to be created.
-            refresh (bool, optional): Whether to refresh the index after the creation. Default is False.
+            **kwargs: Additional keyword arguments.
+                - refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
+                - refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
 
         Raises:
             ConflictError: If a Collection with the same id already exists in the database.
 
+        Returns:
+            None
+
         Notes:
             A new index is created for the items in the Collection using the `create_item_index` function.
         """
         collection_id = collection["id"]
 
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the creation attempt
+        logger.info(f"Creating collection {collection_id} with refresh={refresh}")
+
+        # Check if the collection already exists
         if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
             raise ConflictError(f"Collection {collection_id} already exists")
 
+        # Index the collection in the database
         await self.client.index(
             index=COLLECTIONS_INDEX,
             id=collection_id,
@@ -941,6 +944,7 @@ class DatabaseLogic(BaseDatabaseLogic):
             refresh=refresh,
         )
 
+        # Create the item index for the collection
         await create_item_index(collection_id)
 
     async def find_collection(self, collection_id: str) -> Collection:
@@ -970,29 +974,52 @@ class DatabaseLogic(BaseDatabaseLogic):
         return collection["_source"]
 
     async def update_collection(
-        self, collection_id: str, collection: Collection, refresh: bool = False
+        self, collection_id: str, collection: Collection, **kwargs: Any
     ):
-        """Update a collection from the database.
+        """Update a collection in the database.
 
         Args:
-            self: The instance of the object calling this function.
             collection_id (str): The ID of the collection to be updated.
             collection (Collection): The Collection object to be used for the update.
+            **kwargs: Additional keyword arguments.
+                - refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
+                - refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
+        Returns:
+            None
 
         Raises:
-            NotFoundError: If the collection with the given `collection_id` is not
-            found in the database.
+            NotFoundError: If the collection with the given `collection_id` is not found in the database.
+            ConflictError: If a conflict occurs during the update.
 
         Notes:
             This function updates the collection in the database using the specified
-            `collection_id` and with the collection specified in the `Collection` object.
-            If the collection is not found, a `NotFoundError` is raised.
+            `collection_id` and the provided `Collection` object. If the collection ID
+            changes, the function creates a new collection, reindexes the items, and deletes
+            the old collection.
         """
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the update attempt
+        logger.info(f"Updating collection {collection_id} with refresh={refresh}")
+
+        # Ensure the collection exists
         await self.find_collection(collection_id=collection_id)
 
+        # Handle collection ID change
         if collection_id != collection["id"]:
+            logger.info(
+                f"Collection ID change detected: {collection_id} -> {collection['id']}"
+            )
+
+            # Create the new collection
             await self.create_collection(collection, refresh=refresh)
 
+            # Reindex items from the old collection to the new collection
             await self.client.reindex(
                 body={
                     "dest": {"index": f"{ITEMS_INDEX_PREFIX}{collection['id']}"},
@@ -1006,9 +1033,11 @@ class DatabaseLogic(BaseDatabaseLogic):
                 refresh=refresh,
             )
 
+            # Delete the old collection
             await self.delete_collection(collection_id)
 
         else:
+            # Update the existing collection
             await self.client.index(
                 index=COLLECTIONS_INDEX,
                 id=collection_id,
@@ -1016,33 +1045,57 @@ class DatabaseLogic(BaseDatabaseLogic):
                 refresh=refresh,
             )
 
-    async def delete_collection(self, collection_id: str, refresh: bool = False):
+    async def delete_collection(self, collection_id: str, **kwargs: Any):
         """Delete a collection from the database.
 
         Parameters:
-            self: The instance of the object calling this function.
             collection_id (str): The ID of the collection to be deleted.
-            refresh (bool): Whether to refresh the index after the deletion (default: False).
+            kwargs (Any, optional): Additional keyword arguments, including `refresh`.
+                - refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
+                - refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
 
         Raises:
             NotFoundError: If the collection with the given `collection_id` is not found in the database.
 
+        Returns:
+            None
+
         Notes:
             This function first verifies that the collection with the specified `collection_id` exists in the database, and then
-            deletes the collection. If `refresh` is set to True, the index is refreshed after the deletion. Additionally, this
-            function also calls `delete_item_index` to delete the index for the items in the collection.
+            deletes the collection. If `refresh` is set to "true", "false", or "wait_for", the index is refreshed accordingly after
+            the deletion. Additionally, this function also calls `delete_item_index` to delete the index for the items in the collection.
         """
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Verify that the collection exists
         await self.find_collection(collection_id=collection_id)
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the deletion attempt
+        logger.info(f"Deleting collection {collection_id} with refresh={refresh}")
+
+        # Delete the collection from the database
         await self.client.delete(
             index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh
         )
-        await delete_item_index(collection_id)
+
+        # Delete the item index for the collection
+        try:
+            await delete_item_index(collection_id)
+        except Exception as e:
+            logger.error(
+                f"Failed to delete item index for collection {collection_id}: {e}"
+            )
 
     async def bulk_async(
         self,
         collection_id: str,
         processed_items: List[Item],
-        refresh: bool = False,
+        **kwargs: Any,
     ) -> Tuple[int, List[Dict[str, Any]]]:
         """
         Perform a bulk insert of items into the database asynchronously.
@@ -1050,7 +1103,12 @@ class DatabaseLogic(BaseDatabaseLogic):
         Args:
             collection_id (str): The ID of the collection to which the items belong.
             processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
-            refresh (bool): Whether to refresh the index after the bulk insert (default: False).
+            **kwargs (Any): Additional keyword arguments, including:
+                - refresh (str, optional): Whether to refresh the index after the bulk insert.
+                Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
+                - refresh (bool, optional): Whether to refresh the index after the bulk insert.
+                - raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
+                Defaults to the value of `self.async_settings.raise_on_bulk_error`.
 
         Returns:
             Tuple[int, List[Dict[str, Any]]]: A tuple containing:
@@ -1059,10 +1117,31 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         Notes:
             This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
-            The insert is performed asynchronously, and the event loop is used to run the operation in a separate executor.
-            The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True,
-            the index is refreshed after the bulk insert.
+            The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
+            completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
+            parameter determines whether the index is refreshed after the bulk insert:
+                - "true": Forces an immediate refresh of the index.
+                - "false": Does not refresh the index immediately (default behavior).
+                - "wait_for": Waits for the next refresh cycle to make the changes visible.
         """
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the bulk insert attempt
+        logger.info(
+            f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
+        )
+
+        # Handle empty processed_items
+        if not processed_items:
+            logger.warning(f"No items to insert for collection {collection_id}")
+            return 0, []
+
+        # Perform the bulk insert
         raise_on_error = self.async_settings.raise_on_bulk_error
         success, errors = await helpers.async_bulk(
             self.client,
@@ -1070,13 +1149,19 @@ class DatabaseLogic(BaseDatabaseLogic):
             refresh=refresh,
             raise_on_error=raise_on_error,
         )
+
+        # Log the result
+        logger.info(
+            f"Bulk insert completed for collection {collection_id}: {success} successes, {len(errors)} errors"
+        )
+
         return success, errors
 
     def bulk_sync(
         self,
         collection_id: str,
         processed_items: List[Item],
-        refresh: bool = False,
+        **kwargs: Any,
     ) -> Tuple[int, List[Dict[str, Any]]]:
         """
         Perform a bulk insert of items into the database synchronously.
@@ -1084,7 +1169,12 @@ class DatabaseLogic(BaseDatabaseLogic):
         Args:
             collection_id (str): The ID of the collection to which the items belong.
             processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
-            refresh (bool): Whether to refresh the index after the bulk insert (default: False).
+            **kwargs (Any): Additional keyword arguments, including:
+                - refresh (str, optional): Whether to refresh the index after the bulk insert.
+                Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
+                - refresh (bool, optional): Whether to refresh the index after the bulk insert.
+                - raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
+                Defaults to the value of `self.async_settings.raise_on_bulk_error`.
 
         Returns:
             Tuple[int, List[Dict[str, Any]]]: A tuple containing:
@@ -1094,9 +1184,30 @@ class DatabaseLogic(BaseDatabaseLogic):
         Notes:
             This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
             The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
-            completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to
-            True, the index is refreshed after the bulk insert.
+            completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
+            parameter determines whether the index is refreshed after the bulk insert:
+                - "true": Forces an immediate refresh of the index.
+                - "false": Does not refresh the index immediately (default behavior).
+                - "wait_for": Waits for the next refresh cycle to make the changes visible.
         """
+        # Ensure kwargs is a dictionary
+        kwargs = kwargs or {}
+
+        # Resolve the `refresh` parameter
+        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
+        refresh = validate_refresh(refresh)
+
+        # Log the bulk insert attempt
+        logger.info(
+            f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
+        )
+
+        # Handle empty processed_items
+        if not processed_items:
+            logger.warning(f"No items to insert for collection {collection_id}")
+            return 0, []
+
+        # Perform the bulk insert
         raise_on_error = self.sync_settings.raise_on_bulk_error
         success, errors = helpers.bulk(
             self.sync_client,
@@ -1104,6 +1215,12 @@ class DatabaseLogic(BaseDatabaseLogic):
             refresh=refresh,
             raise_on_error=raise_on_error,
         )
+
+        # Log the result
+        logger.info(
+            f"Bulk insert completed for collection {collection_id}: {success} successes, {len(errors)} errors"
+        )
+
         return success, errors
 
     # DANGER