sfeos-helpers 6.8.1__tar.gz → 6.10.0__tar.gz

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/.gitignore

@@ -141,3 +141,14 @@ venv
 /docs/src/api/*
 
 .DS_Store
+
+# Helm
+*.tgz
+charts/*/charts/
+charts/*/requirements.lock
+charts/*/Chart.lock
+helm-chart/stac-fastapi/charts/
+helm-chart/stac-fastapi/Chart.lock
+helm-chart/stac-fastapi/*.tgz
+helm-chart/test-results/
+helm-chart/tmp/

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sfeos_helpers
-Version: 6.8.1
+Version: 6.10.0
 Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
 Project-URL: Homepage, https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.11
-Requires-Dist: stac-fastapi-core==6.8.1
+Requires-Dist: stac-fastapi-core==6.10.0
 Description-Content-Type: text/markdown
 
 # sfeos-helpers

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/pyproject.toml

@@ -29,7 +29,7 @@ keywords = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "stac-fastapi.core==6.8.1",
+    "stac-fastapi.core==6.10.0",
 ]
 
 [project.urls]

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/aggregation/client.py

@@ -1,12 +1,12 @@
 """Client implementation for the STAC API Aggregation Extension."""
 
-from pathlib import Path
+
 from typing import Annotated, Any, Dict, List, Optional, Union
 from urllib.parse import unquote_plus, urljoin
 
 import attr
 import orjson
-from fastapi import HTTPException, Request
+from fastapi import HTTPException, Path, Request
 from pygeofilter.backends.cql2_json import to_cql2
 from pygeofilter.parsers.cql2_text import parse as parse_cql2_text
 from stac_pydantic.shared import BBox
@@ -316,8 +316,6 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
             search, datetime_search = self.database.apply_datetime_filter(
                 search=search, datetime=aggregate_request.datetime
             )
-        else:
-            datetime_search = {"gte": None, "lte": None}
 
         if aggregate_request.bbox:
             bbox = aggregate_request.bbox
@@ -416,7 +414,7 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
                 geometry_geohash_grid_precision,
                 geometry_geotile_grid_precision,
                 datetime_frequency_interval,
-                datetime_search,
+                aggregate_request.datetime,
             )
         except Exception as error:
             if not isinstance(error, IndexError):

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/database/__init__.py

@@ -30,7 +30,19 @@ Function Naming Conventions:
 """
 
 # Re-export all functions for backward compatibility
-from .datetime import extract_date, extract_first_date_from_index, return_date
+from .catalogs import (
+    search_children_with_pagination_shared,
+    search_collections_by_parent_id_shared,
+    search_sub_catalogs_with_pagination_shared,
+    update_catalog_in_index_shared,
+)
+from .datetime import (
+    extract_date,
+    extract_first_date_from_index,
+    extract_last_date_from_index,
+    is_index_closed,
+    return_date,
+)
 from .document import mk_actions, mk_item_id
 from .index import (
     create_index_templates_shared,
@@ -51,6 +63,11 @@ from .query import (
 from .utils import add_bbox_shape_to_collection, get_bool_env, validate_refresh
 
 __all__ = [
+    # Catalog operations
+    "search_collections_by_parent_id_shared",
+    "search_sub_catalogs_with_pagination_shared",
+    "update_catalog_in_index_shared",
+    "search_children_with_pagination_shared",
     # Index operations
     "create_index_templates_shared",
     "delete_item_index_shared",
@@ -77,4 +94,6 @@ __all__ = [
     "return_date",
     "extract_date",
     "extract_first_date_from_index",
+    "extract_last_date_from_index",
+    "is_index_closed",
 ]

sfeos_helpers-6.10.0/stac_fastapi/sfeos_helpers/database/catalogs.py

@@ -0,0 +1,190 @@
+"""Catalog-related database operations for Elasticsearch/OpenSearch.
+
+This module provides helper functions for catalog operations that require
+direct Elasticsearch/OpenSearch client access. These functions are used by
+the CatalogsExtension to maintain database-agnostic code in the core module.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
+
+logger = logging.getLogger(__name__)
+
+
+async def search_collections_by_parent_id_shared(
+    es_client: Any, catalog_id: str, size: int = 10000
+) -> List[Dict[str, Any]]:
+    """Search for collections that have a specific catalog as a parent.
+
+    Args:
+        es_client: Elasticsearch/OpenSearch client instance.
+        catalog_id: The catalog ID to search for in parent_ids.
+        size: Maximum number of results to return (default: 10000).
+
+    Returns:
+        List of collection documents from the search results.
+    """
+    query_body = {"query": {"term": {"parent_ids": catalog_id}}, "size": size}
+    try:
+        search_result = await es_client.search(index=COLLECTIONS_INDEX, body=query_body)
+        return [hit["_source"] for hit in search_result["hits"]["hits"]]
+    except Exception as e:
+        logger.error(f"Error searching for collections with parent {catalog_id}: {e}")
+        return []
+
+
+async def search_sub_catalogs_with_pagination_shared(
+    es_client: Any,
+    catalog_id: str,
+    limit: int = 10,
+    token: Optional[str] = None,
+) -> tuple[List[Dict[str, Any]], int, Optional[str]]:
+    """Search for sub-catalogs with pagination support.
+
+    Args:
+        es_client: Elasticsearch/OpenSearch client instance.
+        catalog_id: The parent catalog ID.
+        limit: Maximum number of results to return (default: 10).
+        token: Pagination token for cursor-based pagination.
+
+    Returns:
+        Tuple of (catalogs, total_count, next_token).
+    """
+    sort_fields: List[Dict[str, Any]] = [{"id": {"order": "asc"}}]
+    query_body: Dict[str, Any] = {
+        "query": {
+            "bool": {
+                "must": [
+                    {"term": {"parent_ids": catalog_id}},
+                    {"term": {"type": "Catalog"}},
+                ]
+            }
+        },
+        "sort": sort_fields,
+        "size": limit,
+    }
+
+    # Handle pagination cursor (token)
+    # Token format: "value1|value2|..." matching the sort fields
+    if token:
+        try:
+            search_after = token.split("|")
+            if len(search_after) == len(sort_fields):
+                query_body["search_after"] = search_after
+        except Exception:
+            logger.debug(f"Invalid pagination token: {token}")
+
+    # Execute the search
+    try:
+        search_result = await es_client.search(index=COLLECTIONS_INDEX, body=query_body)
+    except Exception as e:
+        logger.error(f"Error searching for catalogs with parent {catalog_id}: {e}")
+        search_result = {"hits": {"hits": []}}
+
+    # Process results
+    hits = search_result.get("hits", {}).get("hits", [])
+    total_hits = search_result.get("hits", {}).get("total", {}).get("value", 0)
+
+    catalogs = [hit["_source"] for hit in hits]
+
+    # Generate next token if more results exist
+    next_token = None
+    if len(hits) == limit and len(catalogs) > 0:
+        last_hit_sort = hits[-1].get("sort")
+        if last_hit_sort:
+            next_token = "|".join(str(x) for x in last_hit_sort)
+
+    return catalogs, total_hits, next_token
+
+
+async def update_catalog_in_index_shared(
+    es_client: Any, catalog_id: str, catalog_data: Dict[str, Any]
+) -> None:
+    """Update a catalog document in the index.
+
+    Args:
+        es_client: Elasticsearch/OpenSearch client instance.
+        catalog_id: The catalog ID.
+        catalog_data: The catalog document to update.
+    """
+    try:
+        await es_client.index(
+            index=COLLECTIONS_INDEX,
+            id=catalog_id,
+            body=catalog_data,
+            refresh=True,
+        )
+    except Exception as e:
+        logger.error(f"Error updating catalog {catalog_id} in index: {e}")
+        raise
+
+
+async def search_children_with_pagination_shared(
+    es_client: Any,
+    catalog_id: str,
+    limit: int = 10,
+    token: Optional[str] = None,
+    resource_type: Optional[str] = None,
+) -> tuple[List[Dict[str, Any]], int, Optional[str]]:
+    """Search for children (catalogs and collections) with pagination.
+
+    Args:
+        es_client: Elasticsearch/OpenSearch client instance.
+        catalog_id: The parent catalog ID.
+        limit: Maximum number of results to return (default: 10).
+        token: Pagination token for cursor-based pagination.
+        resource_type: Optional filter by type (Catalog or Collection).
+
+    Returns:
+        Tuple of (children, total_count, next_token).
+    """
+    # Base filter: Parent match
+    filter_queries = [{"term": {"parent_ids": catalog_id}}]
+
+    # Optional filter: Type
+    if resource_type:
+        filter_queries.append({"term": {"type": resource_type}})
+
+    body = {
+        "query": {"bool": {"filter": filter_queries}},
+        "sort": [{"id": {"order": "asc"}}],
+        "size": limit,
+    }
+
+    # Handle search_after token
+    search_after: Optional[List[str]] = None
+    if token:
+        try:
+            search_after_parts = token.split("|")
+            # If the number of sort fields doesn't match token parts, ignore the token
+            if len(search_after_parts) == len(body["sort"]):  # type: ignore
+                search_after = search_after_parts
+        except Exception:
+            search_after = None
+
+        if search_after is not None:
+            body["search_after"] = search_after
+
+    # Execute search
+    try:
+        search_result = await es_client.search(index=COLLECTIONS_INDEX, body=body)
+    except Exception as e:
+        logger.error(f"Error searching for children of catalog {catalog_id}: {e}")
+        search_result = {"hits": {"hits": []}}
+
+    # Process results
+    hits = search_result.get("hits", {}).get("hits", [])
+    total = search_result.get("hits", {}).get("total", {}).get("value", 0)
+
+    children = [hit["_source"] for hit in hits]
+
+    # Generate next token if more results exist
+    next_token = None
+    if len(hits) == limit:
+        next_token_values = hits[-1].get("sort")
+        if next_token_values:
+            next_token = "|".join(str(val) for val in next_token_values)
+
+    return children, total, next_token

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/database/datetime.py

@@ -145,7 +145,7 @@ def extract_date(date_str: str) -> date:
         date_str: ISO format date string
 
     Returns:
-        A date object extracted from the input string.
+        A date object extracted from the input string or None.
     """
     date_str = date_str.replace("Z", "+00:00")
     return datetime_type.fromisoformat(date_str).date()
@@ -186,3 +186,56 @@ def extract_first_date_from_index(index_name: str) -> date:
         raise ValueError(
             f"Invalid date format in index name '{index_name}': '{date_string}'"
         ) from e
+
+
+def is_index_closed(alias_name: str) -> bool:
+    """Check if an index alias is closed (has two dates indicating a date range).
+
+    A closed index has an alias like 'items_start_datetime_collection_2025-11-06-2025-11-08'
+    indicating a fixed date range that should not be modified.
+
+    Args:
+        alias_name: The alias name to check.
+
+    Returns:
+        True if the alias contains two dates (closed), False if it has one date (open).
+    """
+    date_pattern = r"\d{4}-\d{2}-\d{2}"
+    matches = re.findall(date_pattern, alias_name)
+    return len(matches) >= 2
+
+
+def extract_last_date_from_index(index_name: str) -> date:
+    """Extract the last date from an index name containing date patterns.
+
+    Searches for date patterns (YYYY-MM-DD) within the index name string
+    and returns the last found date as a date object.
+
+    Args:
+        index_name: Index name containing date patterns.
+
+    Returns:
+        A date object extracted from the last date pattern found in the index name.
+    """
+    date_pattern = r"\d{4}-\d{2}-\d{2}"
+    matches = re.findall(date_pattern, index_name)
+
+    if not matches:
+        logger.error(f"No date pattern found in index name: '{index_name}'")
+        raise ValueError(
+            f"No date pattern (YYYY-MM-DD) found in index name: '{index_name}'"
+        )
+
+    date_string = matches[-1]
+
+    try:
+        extracted_date = datetime_type.strptime(date_string, "%Y-%m-%d").date()
+        return extracted_date
+    except ValueError as e:
+        logger.error(
+            f"Invalid date format found in index name '{index_name}': "
+            f"'{date_string}' - {str(e)}"
+        )
+        raise ValueError(
+            f"Invalid date format in index name '{index_name}': '{date_string}'"
+        ) from e

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/database/index.py

@@ -4,11 +4,9 @@ This module provides functions for creating and managing indices in Elasticsearc
 """
 
 import re
-from datetime import datetime
+from datetime import date, datetime
 from functools import lru_cache
-from typing import Any, List, Optional
-
-from dateutil.parser import parse  # type: ignore[import]
+from typing import Any, Dict, List, Optional, Tuple
 
 from stac_fastapi.sfeos_helpers.mappings import (
     _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
@@ -71,54 +69,103 @@ def indices(collection_ids: Optional[List[str]]) -> str:
 
 
 def filter_indexes_by_datetime(
-    indexes: List[str], gte: Optional[str], lte: Optional[str]
+    collection_indexes: List[Tuple[Dict[str, str], ...]],
+    datetime_search: Dict[str, Dict[str, Optional[str]]],
+    use_datetime: bool,
 ) -> List[str]:
-    """Filter indexes based on datetime range extracted from index names.
+    """
+    Filter Elasticsearch index aliases based on datetime search criteria.
+
+    Filters a list of collection indexes by matching their datetime, start_datetime, and end_datetime
+    aliases against the provided search criteria. Each criterion can have optional 'gte' (greater than
+    or equal) and 'lte' (less than or equal) bounds.
 
     Args:
-        indexes: List of index names containing dates
-        gte: Greater than or equal date filter (ISO format, optional 'Z' suffix)
-        lte: Less than or equal date filter (ISO format, optional 'Z' suffix)
+        collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries
+            with 'datetime', 'start_datetime', and 'end_datetime' aliases.
+        datetime_search (Dict[str, Dict[str, Optional[str]]]): A dictionary with keys 'datetime',
+            'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format
+            datetime strings or None.
+        use_datetime (bool): Flag determining which datetime field to filter on:
+            - True: Filters using 'datetime' alias.
+            - False: Filters using 'start_datetime' and 'end_datetime' aliases.
 
     Returns:
-        List of filtered index names
+        List[str]: A list of start_datetime aliases that match all provided search criteria.
     """
 
-    def parse_datetime(dt_str: str) -> datetime:
-        """Parse datetime string, handling both with and without 'Z' suffix."""
-        return parse(dt_str).replace(tzinfo=None)
-
-    def extract_date_range_from_index(index_name: str) -> tuple:
-        """Extract start and end dates from index name."""
-        date_pattern = r"(\d{4}-\d{2}-\d{2})"
-        dates = re.findall(date_pattern, index_name)
-
-        if len(dates) == 1:
-            start_date = datetime.strptime(dates[0], "%Y-%m-%d")
-            max_date = datetime.max.replace(microsecond=0)
-            return start_date, max_date
-        else:
-            start_date = datetime.strptime(dates[0], "%Y-%m-%d")
-            end_date = datetime.strptime(dates[1], "%Y-%m-%d")
-            return start_date, end_date
-
-    def is_index_in_range(
-        start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime
+    def extract_date_from_alias(alias: str) -> Optional[tuple[datetime, datetime]]:
+        date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}")
+        try:
+            dates = date_pattern.findall(alias)
+
+            if not dates:
+                return None
+
+            if len(dates) >= 2:
+                return datetime.strptime(dates[-2], "%Y-%m-%d"), datetime.strptime(
+                    dates[-1], "%Y-%m-%d"
+                )
+            else:
+                date = datetime.strptime(dates[-1], "%Y-%m-%d")
+                return date, date
+        except (ValueError, IndexError):
+            return None
+
+    def parse_search_date(date_str: Optional[str]) -> Optional[date]:
+        if not date_str:
+            return None
+        date_str = date_str.rstrip("Z")
+        return datetime.fromisoformat(date_str).date()
+
+    def check_criteria(
+        value_begin: datetime, value_end: datetime, criteria: Dict
     ) -> bool:
-        """Check if index date range overlaps with filter range."""
-        return not (
-            end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date()
-        )
+        gte = parse_search_date(criteria.get("gte"))
+        lte = parse_search_date(criteria.get("lte"))
+
+        if gte and value_end.date() < gte:
+            return False
+        if lte and value_begin.date() > lte:
+            return False
 
-    gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0)
-    lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0)
+        return True
 
     filtered_indexes = []
 
-    for index in indexes:
-        start_date, end_date = extract_date_range_from_index(index)
-        if is_index_in_range(start_date, end_date, gte_dt, lte_dt):
-            filtered_indexes.append(index)
+    for index_tuple in collection_indexes:
+        if not index_tuple:
+            continue
+
+        index_dict = index_tuple[0]
+        start_datetime_alias = index_dict.get("start_datetime")
+        end_datetime_alias = index_dict.get("end_datetime")
+        datetime_alias = index_dict.get("datetime")
+
+        if start_datetime_alias:
+            start_date = extract_date_from_alias(start_datetime_alias)
+            if not check_criteria(
+                start_date[0], start_date[1], datetime_search.get("start_datetime", {})
+            ):
+                continue
+        if end_datetime_alias:
+            end_date = extract_date_from_alias(end_datetime_alias)
+            if not check_criteria(
+                end_date[0], end_date[1], datetime_search.get("end_datetime", {})
+            ):
+                continue
+        if datetime_alias:
+            datetime_date = extract_date_from_alias(datetime_alias)
+            if not check_criteria(
+                datetime_date[0], datetime_date[1], datetime_search.get("datetime", {})
+            ):
+                continue
+
+        primary_datetime_alias = (
+            datetime_alias if use_datetime else start_datetime_alias
+        )
+
+        filtered_indexes.append(primary_datetime_alias)
 
     return filtered_indexes
 
@@ -177,7 +224,12 @@ async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
     client = settings.create_client
 
     name = index_alias_by_collection_id(collection_id)
-    resolved = await client.indices.resolve_index(name=name, ignore=[404])
+    if hasattr(client, "options"):
+        resolved = await client.options(ignore_status=[404]).indices.resolve_index(
+            name=name
+        )
+    else:
+        resolved = await client.indices.resolve_index(name=name, ignore=[404])
     if "aliases" in resolved and resolved["aliases"]:
         [alias] = resolved["aliases"]
         await client.indices.delete_alias(index=alias["indices"], name=alias["name"])

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/database/query.py

@@ -130,7 +130,7 @@ def apply_collections_datetime_filter_shared(
 
 
 def apply_collections_bbox_filter_shared(
-    bbox: Union[str, List[float], None]
+    bbox: Union[str, List[float], None],
 ) -> Optional[Dict[str, Dict]]:
     """Create a geo_shape filter for collections bbox search.
 

{sfeos_helpers-6.8.1 → sfeos_helpers-6.10.0}/stac_fastapi/sfeos_helpers/database/utils.py

@@ -5,7 +5,7 @@ in Elasticsearch/OpenSearch, such as parameter validation.
 """
 
 import logging
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
 
 from stac_fastapi.core.utilities import bbox2polygon, get_bool_env
 from stac_fastapi.extensions.core.transaction.request import (
@@ -354,10 +354,42 @@ def operations_to_script(operations: List, create_nest: bool = False) -> Dict:
                 commands=commands, operation=operation, path=path, params=params
             )
 
-        source = "".join(commands)
+    source = "".join(commands)
 
     return {
         "source": source,
         "lang": "painless",
         "params": params,
     }
+
+
+def add_hidden_filter(
+    query: Optional[Dict[str, Any]] = None, hide_item_path: Optional[str] = None
+) -> Dict[str, Any]:
+    """Add hidden filter to a query to exclude hidden items.
+
+    Args:
+        query: Optional Elasticsearch query to combine with hidden filter
+        hide_item_path: Path to the hidden field (e.g., "properties._private.hidden")
+                       If None or empty, return original query (no filtering)
+
+    Returns:
+        Query with hidden filter applied
+    """
+    if not hide_item_path:
+        return query or {"match_all": {}}
+
+    hidden_filter = {
+        "bool": {
+            "should": [
+                {"term": {hide_item_path: False}},
+                {"bool": {"must_not": {"exists": {"field": hide_item_path}}}},
+            ],
+            "minimum_should_match": 1,
+        }
+    }
+
+    if query:
+        return {"bool": {"must": [query, hidden_filter]}}
+    else:
+        return hidden_filter