sfeos-helpers 5.0.0a0__py3-none-any.whl

stac_fastapi/sfeos_helpers/database/document.py

@@ -0,0 +1,48 @@
+"""Document operations for Elasticsearch/OpenSearch.
+
+This module provides functions for working with documents in Elasticsearch/OpenSearch,
+including document ID generation and bulk action creation.
+"""
+
+from typing import Any, Dict, List
+
+from stac_fastapi.sfeos_helpers.database.index import index_alias_by_collection_id
+from stac_fastapi.types.stac import Item
+
+
+def mk_item_id(item_id: str, collection_id: str) -> str:
+    """Create the document id for an Item in Elasticsearch.
+
+    Args:
+        item_id (str): The id of the Item.
+        collection_id (str): The id of the Collection that the Item belongs to.
+
+    Returns:
+        str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
+    """
+    return f"{item_id}|{collection_id}"
+
+
+def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
+    """Create Elasticsearch bulk actions for a list of processed items.
+
+    Args:
+        collection_id (str): The identifier for the collection the items belong to.
+        processed_items (List[Item]): The list of processed items to be bulk indexed.
+
+    Returns:
+        List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
+        each action being a dictionary with the following keys:
+        - `_index`: the index to store the document in.
+        - `_id`: the document's identifier.
+        - `_source`: the source of the document.
+    """
+    index_alias = index_alias_by_collection_id(collection_id)
+    return [
+        {
+            "_index": index_alias,
+            "_id": mk_item_id(item["id"], item["collection"]),
+            "_source": item,
+        }
+        for item in processed_items
+    ]

stac_fastapi/sfeos_helpers/database/index.py

@@ -0,0 +1,130 @@
+"""Index management functions for Elasticsearch/OpenSearch.
+
+This module provides functions for creating and managing indices in Elasticsearch/OpenSearch.
+"""
+
+from functools import lru_cache
+from typing import Any, List, Optional
+
+from stac_fastapi.sfeos_helpers.mappings import (
+    _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
+    COLLECTIONS_INDEX,
+    ES_COLLECTIONS_MAPPINGS,
+    ES_ITEMS_MAPPINGS,
+    ES_ITEMS_SETTINGS,
+    ITEM_INDICES,
+    ITEMS_INDEX_PREFIX,
+)
+
+
+@lru_cache(256)
+def index_by_collection_id(collection_id: str) -> str:
+    """
+    Translate a collection id into an Elasticsearch index name.
+
+    Args:
+        collection_id (str): The collection id to translate into an index name.
+
+    Returns:
+        str: The index name derived from the collection id.
+    """
+    cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+    return (
+        f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
+    )
+
+
+@lru_cache(256)
+def index_alias_by_collection_id(collection_id: str) -> str:
+    """
+    Translate a collection id into an Elasticsearch index alias.
+
+    Args:
+        collection_id (str): The collection id to translate into an index alias.
+
+    Returns:
+        str: The index alias derived from the collection id.
+    """
+    cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+    return f"{ITEMS_INDEX_PREFIX}{cleaned}"
+
+
+def indices(collection_ids: Optional[List[str]]) -> str:
+    """
+    Get a comma-separated string of index names for a given list of collection ids.
+
+    Args:
+        collection_ids: A list of collection ids.
+
+    Returns:
+        A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
+    """
+    return (
+        ",".join(map(index_alias_by_collection_id, collection_ids))
+        if collection_ids
+        else ITEM_INDICES
+    )
+
+
+async def create_index_templates_shared(settings: Any) -> None:
+    """Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
+
+    Args:
+        settings (Any): The settings object containing the client configuration.
+            Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+
+    Returns:
+        None: This function doesn't return any value but creates index templates in the database.
+
+    Notes:
+        This function creates two index templates:
+        1. A template for the Collections index with the appropriate mappings
+        2. A template for the Items indices with both settings and mappings
+
+        These templates ensure that any new indices created with matching patterns
+        will automatically have the correct structure.
+    """
+    client = settings.create_client
+    await client.indices.put_index_template(
+        name=f"template_{COLLECTIONS_INDEX}",
+        body={
+            "index_patterns": [f"{COLLECTIONS_INDEX}*"],
+            "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
+        },
+    )
+    await client.indices.put_index_template(
+        name=f"template_{ITEMS_INDEX_PREFIX}",
+        body={
+            "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
+            "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
+        },
+    )
+    await client.close()
+
+
+async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
+    """Delete the index for items in a collection.
+
+    Args:
+        settings (Any): The settings object containing the client configuration.
+            Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+        collection_id (str): The ID of the collection whose items index will be deleted.
+
+    Returns:
+        None: This function doesn't return any value but deletes an item index in the database.
+
+    Notes:
+        This function deletes an item index and its alias. It first resolves the alias to find
+        the actual index name, then deletes both the alias and the index.
+    """
+    client = settings.create_client
+
+    name = index_alias_by_collection_id(collection_id)
+    resolved = await client.indices.resolve_index(name=name)
+    if "aliases" in resolved and resolved["aliases"]:
+        [alias] = resolved["aliases"]
+        await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
+        await client.indices.delete(index=alias["indices"])
+    else:
+        await client.indices.delete(index=name)
+    await client.close()

stac_fastapi/sfeos_helpers/database/mapping.py

@@ -0,0 +1,38 @@
+"""Mapping functions for Elasticsearch/OpenSearch.
+
+This module provides functions for working with Elasticsearch/OpenSearch mappings.
+"""
+
+from typing import Any, Dict
+
+
+async def get_queryables_mapping_shared(
+    mappings: Dict[str, Dict[str, Any]], collection_id: str = "*"
+) -> Dict[str, str]:
+    """Retrieve mapping of Queryables for search.
+
+    Args:
+        mappings (Dict[str, Dict[str, Any]]): The mapping information returned from
+            Elasticsearch/OpenSearch client's indices.get_mapping() method.
+            Expected structure is {index_name: {"mappings": {...}}}.
+        collection_id (str, optional): The id of the Collection the Queryables
+            belongs to. Defaults to "*".
+
+    Returns:
+        Dict[str, str]: A dictionary containing the Queryables mappings, where keys are
+            field names and values are the corresponding paths in the Elasticsearch/OpenSearch
+            document structure.
+    """
+    queryables_mapping = {}
+
+    for mapping in mappings.values():
+        fields = mapping["mappings"].get("properties", {})
+        properties = fields.pop("properties", {}).get("properties", {}).keys()
+
+        for field_key in fields:
+            queryables_mapping[field_key] = field_key
+
+        for property_key in properties:
+            queryables_mapping[property_key] = f"properties.{property_key}"
+
+    return queryables_mapping

stac_fastapi/sfeos_helpers/database/query.py

@@ -0,0 +1,85 @@
+"""Query building functions for Elasticsearch/OpenSearch.
+
+This module provides functions for building and manipulating Elasticsearch/OpenSearch queries.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from stac_fastapi.sfeos_helpers.mappings import Geometry
+
+
+def apply_free_text_filter_shared(
+    search: Any, free_text_queries: Optional[List[str]]
+) -> Any:
+    """Create a free text query for Elasticsearch/OpenSearch.
+
+    Args:
+        search (Any): The search object to apply the query to.
+        free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+    Returns:
+        Any: The search object with the free text query applied, or the original search
+            object if no free_text_queries were provided.
+
+    Notes:
+        This function creates a query_string query that searches for the specified text strings
+        in all properties of the documents. The query strings are joined with OR operators.
+    """
+    if free_text_queries is not None:
+        free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
+        search = search.query(
+            "query_string", query=f'properties.\\*:"{free_text_query_string}"'
+        )
+
+    return search
+
+
+def apply_intersects_filter_shared(
+    intersects: Geometry,
+) -> Dict[str, Dict]:
+    """Create a geo_shape filter for intersecting geometry.
+
+    Args:
+        intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object.
+
+    Returns:
+        Dict[str, Dict]: A dictionary containing the geo_shape filter configuration
+            that can be used with Elasticsearch/OpenSearch Q objects.
+
+    Notes:
+        This function creates a geo_shape filter configuration to find documents that intersect
+        with the specified geometry. The returned dictionary should be wrapped in a Q object
+        when applied to a search.
+    """
+    return {
+        "geo_shape": {
+            "geometry": {
+                "shape": {
+                    "type": intersects.type.lower(),
+                    "coordinates": intersects.coordinates,
+                },
+                "relation": "intersects",
+            }
+        }
+    }
+
+
+def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
+    """Create a sort configuration for Elasticsearch/OpenSearch queries.
+
+    Args:
+        sortby (List): A list of sort specifications, each containing a field and direction.
+
+    Returns:
+        Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+            configurations, or None if no sort was specified.
+
+    Notes:
+        This function transforms a list of sort specifications into the format required by
+        Elasticsearch/OpenSearch for sorting query results. The returned dictionary can be
+        directly used in search requests.
+    """
+    if sortby:
+        return {s.field: {"order": s.direction} for s in sortby}
+    else:
+        return None

stac_fastapi/sfeos_helpers/database/utils.py

@@ -0,0 +1,50 @@
+"""Utility functions for database operations in Elasticsearch/OpenSearch.
+
+This module provides utility functions for working with database operations
+in Elasticsearch/OpenSearch, such as parameter validation.
+"""
+
+import logging
+from typing import Union
+
+from stac_fastapi.core.utilities import get_bool_env
+
+
+def validate_refresh(value: Union[str, bool]) -> str:
+    """
+    Validate the `refresh` parameter value.
+
+    Args:
+        value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
+
+    Returns:
+        str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
+    """
+    logger = logging.getLogger(__name__)
+
+    # Handle boolean-like values using get_bool_env
+    if isinstance(value, bool) or value in {
+        "true",
+        "false",
+        "1",
+        "0",
+        "yes",
+        "no",
+        "y",
+        "n",
+    }:
+        is_true = get_bool_env("DATABASE_REFRESH", default=value)
+        return "true" if is_true else "false"
+
+    # Normalize to lowercase for case-insensitivity
+    value = value.lower()
+
+    # Handle "wait_for" explicitly
+    if value == "wait_for":
+        return "wait_for"
+
+    # Log a warning for invalid values and default to "false"
+    logger.warning(
+        f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
+    )
+    return "false"

stac_fastapi/sfeos_helpers/filter/__init__.py

@@ -0,0 +1,44 @@
+"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends.
+
+This module provides shared functionality for implementing the STAC API Filter Extension
+with Elasticsearch and OpenSearch. It includes:
+
+1. Functions for converting CQL2 queries to Elasticsearch/OpenSearch query DSL
+2. Helper functions for field mapping and query transformation
+3. Base implementation of the AsyncBaseFiltersClient for Elasticsearch/OpenSearch
+
+The filter package is organized as follows:
+- cql2.py: CQL2 pattern conversion helpers
+- transform.py: Query transformation functions
+- client.py: Filter client implementation
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
+
+from .client import EsAsyncBaseFiltersClient
+
+# Re-export the main functions and classes for backward compatibility
+from .cql2 import (
+    _replace_like_patterns,
+    cql2_like_patterns,
+    cql2_like_to_es,
+    valid_like_substitutions,
+)
+from .transform import to_es, to_es_field
+
+__all__ = [
+    "cql2_like_patterns",
+    "valid_like_substitutions",
+    "cql2_like_to_es",
+    "_replace_like_patterns",
+    "to_es_field",
+    "to_es",
+    "EsAsyncBaseFiltersClient",
+]

stac_fastapi/sfeos_helpers/filter/client.py

@@ -0,0 +1,98 @@
+"""Filter client implementation for Elasticsearch/OpenSearch."""
+
+from collections import deque
+from typing import Any, Dict, Optional
+
+import attr
+
+from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
+from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES
+from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
+from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON
+
+
+@attr.s
+class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
+    """Defines a pattern for implementing the STAC filter extension."""
+
+    database: BaseDatabaseLogic = attr.ib()
+
+    async def get_queryables(
+        self, collection_id: Optional[str] = None, **kwargs
+    ) -> Dict[str, Any]:
+        """Get the queryables available for the given collection_id.
+
+        If collection_id is None, returns the intersection of all
+        queryables over all collections.
+
+        This base implementation returns a blank queryable schema. This is not allowed
+        under OGC CQL but it is allowed by the STAC API Filter Extension
+
+        https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
+
+        Args:
+            collection_id (str, optional): The id of the collection to get queryables for.
+            **kwargs: additional keyword arguments
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the queryables for the given collection.
+        """
+        queryables: Dict[str, Any] = {
+            "$schema": "https://json-schema.org/draft/2019-09/schema",
+            "$id": "https://stac-api.example.com/queryables",
+            "type": "object",
+            "title": "Queryables for STAC API",
+            "description": "Queryable names for the STAC API Item Search filter.",
+            "properties": DEFAULT_QUERYABLES,
+            "additionalProperties": True,
+        }
+        if not collection_id:
+            return queryables
+
+        properties: Dict[str, Any] = queryables["properties"]
+        queryables.update(
+            {
+                "properties": properties,
+                "additionalProperties": False,
+            }
+        )
+
+        mapping_data = await self.database.get_items_mapping(collection_id)
+        mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
+        stack = deque(mapping_properties.items())
+
+        while stack:
+            field_name, field_def = stack.popleft()
+
+            # Iterate over nested fields
+            field_properties = field_def.get("properties")
+            if field_properties:
+                # Fields in Item Properties should be exposed with their un-prefixed names,
+                # and not require expressions to prefix them with properties,
+                # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
+                if field_name == "properties":
+                    stack.extend(field_properties.items())
+                else:
+                    stack.extend(
+                        (f"{field_name}.{k}", v) for k, v in field_properties.items()
+                    )
+
+            # Skip non-indexed or disabled fields
+            field_type = field_def.get("type")
+            if not field_type or not field_def.get("enabled", True):
+                continue
+
+            # Generate field properties
+            field_result = DEFAULT_QUERYABLES.get(field_name, {})
+            properties[field_name] = field_result
+
+            field_name_human = field_name.replace("_", " ").title()
+            field_result.setdefault("title", field_name_human)
+
+            field_type_json = ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
+            field_result.setdefault("type", field_type_json)
+
+            if field_type in {"date", "date_nanos"}:
+                field_result.setdefault("format", "date-time")
+
+        return queryables

stac_fastapi/sfeos_helpers/filter/cql2.py

@@ -0,0 +1,39 @@
+"""CQL2 pattern conversion helpers for Elasticsearch/OpenSearch."""
+
+import re
+
+cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
+valid_like_substitutions = {
+    "\\\\": "\\",
+    "\\%": "%",
+    "\\_": "_",
+    "%": "*",
+    "_": "?",
+}
+
+
+def _replace_like_patterns(match: re.Match) -> str:
+    pattern = match.group()
+    try:
+        return valid_like_substitutions[pattern]
+    except KeyError:
+        raise ValueError(f"'{pattern}' is not a valid escape sequence")
+
+
+def cql2_like_to_es(string: str) -> str:
+    """
+    Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
+
+    Args:
+        string (str): The string containing CQL2 wildcard characters.
+
+    Returns:
+        str: The converted string with Elasticsearch compatible wildcards.
+
+    Raises:
+        ValueError: If an invalid escape sequence is encountered.
+    """
+    return cql2_like_patterns.sub(
+        repl=_replace_like_patterns,
+        string=string,
+    )

stac_fastapi/sfeos_helpers/filter/transform.py

@@ -0,0 +1,133 @@
+"""Query transformation functions for Elasticsearch/OpenSearch."""
+
+from typing import Any, Dict
+
+from stac_fastapi.core.extensions.filter import (
+    AdvancedComparisonOp,
+    ComparisonOp,
+    LogicalOp,
+    SpatialOp,
+)
+
+from .cql2 import cql2_like_to_es
+
+
+def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
+    """
+    Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
+
+    Args:
+        field (str): The field name from a user query or filter.
+
+    Returns:
+        str: The mapped field name suitable for Elasticsearch queries.
+    """
+    return queryables_mapping.get(field, field)
+
+
+def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
+
+    Args:
+        query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
+
+    Returns:
+        Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
+    """
+    if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
+        bool_type = {
+            LogicalOp.AND: "must",
+            LogicalOp.OR: "should",
+            LogicalOp.NOT: "must_not",
+        }[query["op"]]
+        return {
+            "bool": {
+                bool_type: [
+                    to_es(queryables_mapping, sub_query) for sub_query in query["args"]
+                ]
+            }
+        }
+
+    elif query["op"] in [
+        ComparisonOp.EQ,
+        ComparisonOp.NEQ,
+        ComparisonOp.LT,
+        ComparisonOp.LTE,
+        ComparisonOp.GT,
+        ComparisonOp.GTE,
+    ]:
+        range_op = {
+            ComparisonOp.LT: "lt",
+            ComparisonOp.LTE: "lte",
+            ComparisonOp.GT: "gt",
+            ComparisonOp.GTE: "gte",
+        }
+
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        value = query["args"][1]
+        if isinstance(value, dict) and "timestamp" in value:
+            value = value["timestamp"]
+            if query["op"] == ComparisonOp.EQ:
+                return {"range": {field: {"gte": value, "lte": value}}}
+            elif query["op"] == ComparisonOp.NEQ:
+                return {
+                    "bool": {
+                        "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
+                    }
+                }
+            else:
+                return {"range": {field: {range_op[query["op"]]: value}}}
+        else:
+            if query["op"] == ComparisonOp.EQ:
+                return {"term": {field: value}}
+            elif query["op"] == ComparisonOp.NEQ:
+                return {"bool": {"must_not": [{"term": {field: value}}]}}
+            else:
+                return {"range": {field: {range_op[query["op"]]: value}}}
+
+    elif query["op"] == ComparisonOp.IS_NULL:
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        return {"bool": {"must_not": {"exists": {"field": field}}}}
+
+    elif query["op"] == AdvancedComparisonOp.BETWEEN:
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        gte, lte = query["args"][1], query["args"][2]
+        if isinstance(gte, dict) and "timestamp" in gte:
+            gte = gte["timestamp"]
+        if isinstance(lte, dict) and "timestamp" in lte:
+            lte = lte["timestamp"]
+        return {"range": {field: {"gte": gte, "lte": lte}}}
+
+    elif query["op"] == AdvancedComparisonOp.IN:
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        values = query["args"][1]
+        if not isinstance(values, list):
+            raise ValueError(f"Arg {values} is not a list")
+        return {"terms": {field: values}}
+
+    elif query["op"] == AdvancedComparisonOp.LIKE:
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        pattern = cql2_like_to_es(query["args"][1])
+        return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
+
+    elif query["op"] in [
+        SpatialOp.S_INTERSECTS,
+        SpatialOp.S_CONTAINS,
+        SpatialOp.S_WITHIN,
+        SpatialOp.S_DISJOINT,
+    ]:
+        field = to_es_field(queryables_mapping, query["args"][0]["property"])
+        geometry = query["args"][1]
+
+        relation_mapping = {
+            SpatialOp.S_INTERSECTS: "intersects",
+            SpatialOp.S_CONTAINS: "contains",
+            SpatialOp.S_WITHIN: "within",
+            SpatialOp.S_DISJOINT: "disjoint",
+        }
+
+        relation = relation_mapping[query["op"]]
+        return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
+
+    return {}