stac-fastapi-elasticsearch 6.3.0__py3-none-any.whl → 6.4.0__py3-none-any.whl

stac_fastapi/elasticsearch/app.py

@@ -34,7 +34,7 @@ from stac_fastapi.elasticsearch.database_logic import (
     create_collection_index,
     create_index_templates,
 )
-from stac_fastapi.extensions.core import (
+from stac_fastapi.extensions.core import (  # CollectionSearchFilterExtension,
     AggregationExtension,
     CollectionSearchExtension,
     FilterExtension,
@@ -45,6 +45,7 @@ from stac_fastapi.extensions.core import (
 )
 from stac_fastapi.extensions.core.fields import FieldsConformanceClasses
 from stac_fastapi.extensions.core.filter import FilterConformanceClasses
+from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses
 from stac_fastapi.extensions.core.query import QueryConformanceClasses
 from stac_fastapi.extensions.core.sort import SortConformanceClasses
 from stac_fastapi.extensions.third_party import BulkTransactionExtension
@@ -55,7 +56,9 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True)
+ENABLE_COLLECTIONS_SEARCH = get_bool_env("ENABLE_COLLECTIONS_SEARCH", default=True)
 logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS)
+logger.info("ENABLE_COLLECTIONS_SEARCH is set to %s", ENABLE_COLLECTIONS_SEARCH)
 
 settings = ElasticsearchSettings()
 session = Session.create_from_settings(settings)
@@ -70,14 +73,6 @@ filter_extension.conformance_classes.append(
     FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS
 )
 
-# Adding collection search extension for compatibility with stac-auth-proxy
-# (https://github.com/developmentseed/stac-auth-proxy)
-# The extension is not fully implemented yet but is required for collection filtering support
-collection_search_extension = CollectionSearchExtension()
-collection_search_extension.conformance_classes.append(
-    "https://api.stacspec.org/v1.0.0-rc.1/collection-search#filter"
-)
-
 aggregation_extension = AggregationExtension(
     client=EsAsyncBaseAggregationClient(
         database=database_logic, session=session, settings=settings
@@ -96,7 +91,6 @@ search_extensions = [
     TokenPaginationExtension(),
     filter_extension,
     FreeTextExtension(),
-    collection_search_extension,
 ]
 
 if TRANSACTIONS_EXTENSIONS:
@@ -122,6 +116,27 @@ if TRANSACTIONS_EXTENSIONS:
 
 extensions = [aggregation_extension] + search_extensions
 
+# Create collection search extensions if enabled
+if ENABLE_COLLECTIONS_SEARCH:
+    # Create collection search extensions
+    collection_search_extensions = [
+        # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]),
+        SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]),
+        FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]),
+        # CollectionSearchFilterExtension(
+        #     conformance_classes=[FilterConformanceClasses.COLLECTIONS]
+        # ),
+        FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]),
+    ]
+
+    # Initialize collection search with its extensions
+    collection_search_ext = CollectionSearchExtension.from_extensions(
+        collection_search_extensions
+    )
+    collections_get_request_model = collection_search_ext.GET
+
+    extensions.append(collection_search_ext)
+
 database_logic.extensions = [type(ext).__name__ for ext in extensions]
 
 post_request_model = create_post_request_model(search_extensions)
@@ -160,6 +175,10 @@ app_config = {
     "route_dependencies": get_route_dependencies(),
 }
 
+# Add collections_get_request_model if collection search is enabled
+if ENABLE_COLLECTIONS_SEARCH:
+    app_config["collections_get_request_model"] = collections_get_request_model
+
 api = StacApi(**app_config)
 
 

stac_fastapi/elasticsearch/database_logic.py

@@ -17,7 +17,7 @@ from starlette.requests import Request
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
+from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
 from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
 from stac_fastapi.elasticsearch.config import (
     ElasticsearchSettings as SyncElasticsearchSettings,
@@ -170,28 +170,96 @@ class DatabaseLogic(BaseDatabaseLogic):
     """CORE LOGIC"""
 
     async def get_all_collections(
-        self, token: Optional[str], limit: int, request: Request
+        self,
+        token: Optional[str],
+        limit: int,
+        request: Request,
+        sort: Optional[List[Dict[str, Any]]] = None,
+        q: Optional[List[str]] = None,
     ) -> Tuple[List[Dict[str, Any]], Optional[str]]:
-        """Retrieve a list of all collections from Elasticsearch, supporting pagination.
+        """Retrieve a list of collections from Elasticsearch, supporting pagination.
 
         Args:
             token (Optional[str]): The pagination token.
             limit (int): The number of results to return.
+            request (Request): The FastAPI request object.
+            sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request.
+            q (Optional[List[str]]): Free text search terms.
 
         Returns:
             A tuple of (collections, next pagination token if any).
+
+        Raises:
+            HTTPException: If sorting is requested on a field that is not sortable.
         """
-        search_after = None
+        # Define sortable fields based on the ES_COLLECTIONS_MAPPINGS
+        sortable_fields = ["id", "extent.temporal.interval", "temporal"]
+
+        # Format the sort parameter
+        formatted_sort = []
+        if sort:
+            for item in sort:
+                field = item.get("field")
+                direction = item.get("direction", "asc")
+                if field:
+                    # Validate that the field is sortable
+                    if field not in sortable_fields:
+                        raise HTTPException(
+                            status_code=400,
+                            detail=f"Field '{field}' is not sortable. Sortable fields are: {', '.join(sortable_fields)}. "
+                            + "Text fields are not sortable by default in Elasticsearch. "
+                            + "To make a field sortable, update the mapping to use 'keyword' type or add a '.keyword' subfield. ",
+                        )
+                    formatted_sort.append({field: {"order": direction}})
+            # Always include id as a secondary sort to ensure consistent pagination
+            if not any("id" in item for item in formatted_sort):
+                formatted_sort.append({"id": {"order": "asc"}})
+        else:
+            formatted_sort = [{"id": {"order": "asc"}}]
+
+        body = {
+            "sort": formatted_sort,
+            "size": limit,
+        }
+
         if token:
-            search_after = [token]
+            body["search_after"] = [token]
+
+        # Apply free text query if provided
+        if q:
+            # For collections, we want to search across all relevant fields
+            should_clauses = []
+
+            # For each search term
+            for term in q:
+                # Create a multi_match query for each term
+                for field in [
+                    "id",
+                    "title",
+                    "description",
+                    "keywords",
+                    "summaries.platform",
+                    "summaries.constellation",
+                    "providers.name",
+                    "providers.url",
+                ]:
+                    should_clauses.append(
+                        {
+                            "wildcard": {
+                                field: {"value": f"*{term}*", "case_insensitive": True}
+                            }
+                        }
+                    )
 
+            # Add the query to the body using bool query with should clauses
+            body["query"] = {
+                "bool": {"should": should_clauses, "minimum_should_match": 1}
+            }
+
+        # Execute the search
         response = await self.client.search(
             index=COLLECTIONS_INDEX,
-            body={
-                "sort": [{"id": {"order": "asc"}}],
-                "size": limit,
-                **({"search_after": search_after} if search_after is not None else {}),
-            },
+            body=body,
         )
 
         hits = response["hits"]["hits"]
@@ -204,7 +272,9 @@ class DatabaseLogic(BaseDatabaseLogic):
 
         next_token = None
         if len(hits) == limit:
-            next_token = hits[-1]["sort"][0]
+            next_token_values = hits[-1].get("sort")
+            if next_token_values:
+                next_token = next_token_values[0]
 
         return collections, next_token
 
@@ -289,26 +359,99 @@ class DatabaseLogic(BaseDatabaseLogic):
         Returns:
             The filtered search object.
         """
+        # USE_DATETIME env var
+        # True: Search by datetime, if null search by start/end datetime
+        # False: Always search only by start/end datetime
+        USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
+
         datetime_search = return_date(datetime)
 
         if not datetime_search:
             return search, datetime_search
 
-        if "eq" in datetime_search:
-            # For exact matches, include:
-            # 1. Items with matching exact datetime
-            # 2. Items with datetime:null where the time falls within their range
-            should = [
-                Q(
-                    "bool",
-                    filter=[
-                        Q("exists", field="properties.datetime"),
-                        Q("term", **{"properties__datetime": datetime_search["eq"]}),
-                    ],
-                ),
-                Q(
+        if USE_DATETIME:
+            if "eq" in datetime_search:
+                # For exact matches, include:
+                # 1. Items with matching exact datetime
+                # 2. Items with datetime:null where the time falls within their range
+                should = [
+                    Q(
+                        "bool",
+                        filter=[
+                            Q("exists", field="properties.datetime"),
+                            Q(
+                                "term",
+                                **{"properties__datetime": datetime_search["eq"]},
+                            ),
+                        ],
+                    ),
+                    Q(
+                        "bool",
+                        must_not=[Q("exists", field="properties.datetime")],
+                        filter=[
+                            Q("exists", field="properties.start_datetime"),
+                            Q("exists", field="properties.end_datetime"),
+                            Q(
+                                "range",
+                                properties__start_datetime={
+                                    "lte": datetime_search["eq"]
+                                },
+                            ),
+                            Q(
+                                "range",
+                                properties__end_datetime={"gte": datetime_search["eq"]},
+                            ),
+                        ],
+                    ),
+                ]
+            else:
+                # For date ranges, include:
+                # 1. Items with datetime in the range
+                # 2. Items with datetime:null that overlap the search range
+                should = [
+                    Q(
+                        "bool",
+                        filter=[
+                            Q("exists", field="properties.datetime"),
+                            Q(
+                                "range",
+                                properties__datetime={
+                                    "gte": datetime_search["gte"],
+                                    "lte": datetime_search["lte"],
+                                },
+                            ),
+                        ],
+                    ),
+                    Q(
+                        "bool",
+                        must_not=[Q("exists", field="properties.datetime")],
+                        filter=[
+                            Q("exists", field="properties.start_datetime"),
+                            Q("exists", field="properties.end_datetime"),
+                            Q(
+                                "range",
+                                properties__start_datetime={
+                                    "lte": datetime_search["lte"]
+                                },
+                            ),
+                            Q(
+                                "range",
+                                properties__end_datetime={
+                                    "gte": datetime_search["gte"]
+                                },
+                            ),
+                        ],
+                    ),
+                ]
+
+            return (
+                search.query(Q("bool", should=should, minimum_should_match=1)),
+                datetime_search,
+            )
+        else:
+            if "eq" in datetime_search:
+                filter_query = Q(
                     "bool",
-                    must_not=[Q("exists", field="properties.datetime")],
                     filter=[
                         Q("exists", field="properties.start_datetime"),
                         Q("exists", field="properties.end_datetime"),
@@ -321,29 +464,10 @@ class DatabaseLogic(BaseDatabaseLogic):
                             properties__end_datetime={"gte": datetime_search["eq"]},
                         ),
                     ],
-                ),
-            ]
-        else:
-            # For date ranges, include:
-            # 1. Items with datetime in the range
-            # 2. Items with datetime:null that overlap the search range
-            should = [
-                Q(
-                    "bool",
-                    filter=[
-                        Q("exists", field="properties.datetime"),
-                        Q(
-                            "range",
-                            properties__datetime={
-                                "gte": datetime_search["gte"],
-                                "lte": datetime_search["lte"],
-                            },
-                        ),
-                    ],
-                ),
-                Q(
+                )
+            else:
+                filter_query = Q(
                     "bool",
-                    must_not=[Q("exists", field="properties.datetime")],
                     filter=[
                         Q("exists", field="properties.start_datetime"),
                         Q("exists", field="properties.end_datetime"),
@@ -356,13 +480,8 @@ class DatabaseLogic(BaseDatabaseLogic):
                             properties__end_datetime={"gte": datetime_search["gte"]},
                         ),
                     ],
-                ),
-            ]
-
-        return (
-            search.query(Q("bool", should=should, minimum_should_match=1)),
-            datetime_search,
-        )
+                )
+            return search.query(filter_query), datetime_search
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):

stac_fastapi/elasticsearch/version.py

@@ -1,2 +1,2 @@
 """library version."""
-__version__ = "6.3.0"
+__version__ = "6.4.0"

{stac_fastapi_elasticsearch-6.3.0.dist-info → stac_fastapi_elasticsearch-6.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stac-fastapi-elasticsearch
-Version: 6.3.0
+Version: 6.4.0
 Summary: An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT
@@ -15,8 +15,8 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: stac-fastapi-core==6.3.0
-Requires-Dist: sfeos-helpers==6.3.0
+Requires-Dist: stac-fastapi-core==6.4.0
+Requires-Dist: sfeos-helpers==6.4.0
 Requires-Dist: elasticsearch[async]~=8.18.0
 Requires-Dist: uvicorn~=0.23.0
 Requires-Dist: starlette<0.36.0,>=0.35.0
@@ -72,11 +72,10 @@ SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable AP
 - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities
 - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries
 - **Perform geospatial aggregations** to analyze data distribution across space and time
+- **Enhanced collection search capabilities** with support for sorting and field selection
 
 This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs.
 
-
-
 ## Common Deployment Patterns
 
 stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs:
@@ -108,6 +107,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Common Deployment Patterns](#common-deployment-patterns)
   - [Technologies](#technologies)
   - [Table of Contents](#table-of-contents)
+  - [Collection Search Extensions](#collection-search-extensions)
   - [Documentation \& Resources](#documentation--resources)
   - [Package Structure](#package-structure)
   - [Examples](#examples)
@@ -149,6 +149,37 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions
   - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers
 
+## Collection Search Extensions
+
+SFEOS implements extended capabilities for the `/collections` endpoint, allowing for more powerful collection discovery:
+
+- **Sorting**: Sort collections by sortable fields using the `sortby` parameter
+  - Example: `/collections?sortby=+id` (ascending sort by ID)
+  - Example: `/collections?sortby=-id` (descending sort by ID)
+  - Example: `/collections?sortby=-temporal` (descending sort by temporal extent)
+
+- **Field Selection**: Request only specific fields to be returned using the `fields` parameter
+  - Example: `/collections?fields=id,title,description`
+  - This helps reduce payload size when only certain fields are needed
+
+- **Free Text Search**: Search across collection text fields using the `q` parameter
+  - Example: `/collections?q=landsat`
+  - Searches across multiple text fields including title, description, and keywords
+  - Supports partial word matching and relevance-based sorting
+
+These extensions make it easier to build user interfaces that display and navigate through collections efficiently.
+
+> **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled.
+
+> **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on:
+> - `id` (keyword field)
+> - `extent.temporal.interval` (date field)
+> - `temporal` (alias to extent.temporal.interval)
+>
+> Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings.
+>
+> **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable.
+
 ## Package Structure
 
 This project is organized into several packages, each with a specific purpose:
@@ -279,10 +310,12 @@ You can customize additional settings in your `.env` file:
 | `ENABLE_DIRECT_RESPONSE`     | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false`                  | Optional                       |
 | `RAISE_ON_BULK_ERROR`        | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
 | `DATABASE_REFRESH`           | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
+| `ENABLE_COLLECTIONS_SEARCH`  | Enable collection search extensions (sort, fields).                                 | `true`                   | Optional                                                                                    |
 | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
 | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
 | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
 | `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
+| `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | True | Optional |
 
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
@@ -424,6 +457,10 @@ The system uses a precise naming convention:
 - **Root Path Configuration**: The application root path is the base URL by default.
   - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`)
 
+- **Feature Configuration**: Control which features are enabled:
+  - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable.
+  - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable.
+
 
 ## Collection Pagination
 

stac_fastapi_elasticsearch-6.4.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
+stac_fastapi/elasticsearch/app.py,sha256=BXoewlnMDHU6bwBJY9bV02Gho-ZzmIyESMOUWPMuR9k,7506
+stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
+stac_fastapi/elasticsearch/database_logic.py,sha256=6dvex-sEMsl3rBnfTKYey7gyG63wq6KPzOUYRM20r4I,63574
+stac_fastapi/elasticsearch/version.py,sha256=1xfRb7s1Stv055oEewNJpZn9OQUUcaJvgSzCsYLHYSQ,45
+stac_fastapi_elasticsearch-6.4.0.dist-info/METADATA,sha256=-kQ6Zw1DMLH7WOvdtMCqsWbrC7HxnZ1Jp9xPVpJrqg0,39686
+stac_fastapi_elasticsearch-6.4.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+stac_fastapi_elasticsearch-6.4.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
+stac_fastapi_elasticsearch-6.4.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
+stac_fastapi_elasticsearch-6.4.0.dist-info/RECORD,,

stac_fastapi_elasticsearch-6.3.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
-stac_fastapi/elasticsearch/app.py,sha256=42586pNMswKIyC-Q9Lz3hJh88Gm60ibbOsBOpbD0x50,6579
-stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
-stac_fastapi/elasticsearch/database_logic.py,sha256=M6xIgtl3HB9p2V6wZPB6xm6cY02iYa2oG2OJRUbpA0c,58488
-stac_fastapi/elasticsearch/version.py,sha256=rBLPQyvMDNA0PA0jXfByTouJPJn5p0wXiqmUWJMIfYc,45
-stac_fastapi_elasticsearch-6.3.0.dist-info/METADATA,sha256=ijEkw05ul0-pg5rRfPMG0oKnXQd2iRNzvaIW4J4cSBo,36626
-stac_fastapi_elasticsearch-6.3.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-stac_fastapi_elasticsearch-6.3.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
-stac_fastapi_elasticsearch-6.3.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
-stac_fastapi_elasticsearch-6.3.0.dist-info/RECORD,,