stac-fastapi-core 6.9.0__tar.gz → 6.10.1__tar.gz

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/.gitignore

@@ -141,3 +141,14 @@ venv
 /docs/src/api/*
 
 .DS_Store
+
+# Helm
+*.tgz
+charts/*/charts/
+charts/*/requirements.lock
+charts/*/Chart.lock
+helm-chart/stac-fastapi/charts/
+helm-chart/stac-fastapi/Chart.lock
+helm-chart/stac-fastapi/*.tgz
+helm-chart/test-results/
+helm-chart/tmp/

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stac_fastapi_core
-Version: 6.9.0
+Version: 6.10.1
 Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
 Project-URL: Homepage, https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/stac_fastapi/core/core.py

@@ -546,18 +546,20 @@ class CoreClient(AsyncBaseCoreClient):
         return await self.all_collections(
             limit=search_request.limit if hasattr(search_request, "limit") else None,
             bbox=search_request.bbox if hasattr(search_request, "bbox") else None,
-            datetime=search_request.datetime
-            if hasattr(search_request, "datetime")
-            else None,
+            datetime=(
+                search_request.datetime if hasattr(search_request, "datetime") else None
+            ),
             token=search_request.token if hasattr(search_request, "token") else None,
             fields=fields,
             sortby=sortby,
-            filter_expr=search_request.filter
-            if hasattr(search_request, "filter")
-            else None,
-            filter_lang=search_request.filter_lang
-            if hasattr(search_request, "filter_lang")
-            else None,
+            filter_expr=(
+                search_request.filter if hasattr(search_request, "filter") else None
+            ),
+            filter_lang=(
+                search_request.filter_lang
+                if hasattr(search_request, "filter_lang")
+                else None
+            ),
             query=search_request.query if hasattr(search_request, "query") else None,
             q=search_request.q if hasattr(search_request, "q") else None,
             request=request,
@@ -1004,14 +1006,31 @@ class TransactionsClient(AsyncBaseTransactionsClient):
                 database=self.database, settings=self.settings
             )
             features = item_dict["features"]
-            processed_items = [
+            all_prepped = [
                 bulk_client.preprocess_item(
                     feature, base_url, BulkTransactionMethod.INSERT
                 )
                 for feature in features
             ]
+            # Filter out None values (skipped duplicates from DB check)
+            processed_items = [item for item in all_prepped if item is not None]
+            skipped_db_duplicates = len(all_prepped) - len(processed_items)
+
+            # Deduplicate items within the batch by ID (keep last occurrence)
+            # This matches ES behavior where later items overwrite earlier ones
+            seen_ids: dict = {}
+            for item in processed_items:
+                seen_ids[item["id"]] = item
+            unique_items = list(seen_ids.values())
+            skipped_batch_duplicates = len(processed_items) - len(unique_items)
+            processed_items = unique_items
+
+            skipped = skipped_db_duplicates + skipped_batch_duplicates
             attempted = len(processed_items)
 
+            if not processed_items:
+                return f"No items to insert. {skipped} items were skipped (duplicates)."
+
             success, errors = await self.database.bulk_async(
                 collection_id=collection_id,
                 processed_items=processed_items,
@@ -1025,7 +1044,7 @@ class TransactionsClient(AsyncBaseTransactionsClient):
                 logger.info(
                     f"Bulk async operation succeeded with {success} actions for collection {collection_id}."
                 )
-            return f"Successfully added {success} Items. {attempted - success} errors occurred."
+            return f"Successfully added {success} Items. {skipped} skipped (duplicates). {attempted - success} errors occurred."
 
         # Handle single item
         await self.database.create_item(
@@ -1338,18 +1357,35 @@ class BulkTransactionsClient(BaseBulkTransactionsClient):
             base_url = ""
 
         processed_items = []
+        skipped_db_duplicates = 0
         for item in items.items.values():
             try:
                 validated = Item(**item) if not isinstance(item, Item) else item
-                processed_items.append(
-                    self.preprocess_item(
-                        validated.model_dump(mode="json"), base_url, items.method
-                    )
+                prepped = self.preprocess_item(
+                    validated.model_dump(mode="json"), base_url, items.method
                 )
+                if prepped is not None:
+                    processed_items.append(prepped)
+                else:
+                    skipped_db_duplicates += 1
             except ValidationError:
                 # Immediately raise on the first invalid item (strict mode)
                 raise
 
+        # Deduplicate items within the batch by ID (keep last occurrence)
+        # This matches ES behavior where later items overwrite earlier ones
+        seen_ids: dict = {}
+        for item in processed_items:
+            seen_ids[item["id"]] = item
+        unique_items = list(seen_ids.values())
+        skipped_batch_duplicates = len(processed_items) - len(unique_items)
+        processed_items = unique_items
+
+        skipped = skipped_db_duplicates + skipped_batch_duplicates
+
+        if not processed_items:
+            return f"No items to insert. {skipped} items were skipped (duplicates)."
+
         collection_id = processed_items[0]["collection"]
         attempted = len(processed_items)
         success, errors = self.database.bulk_sync(
@@ -1362,4 +1398,4 @@ class BulkTransactionsClient(BaseBulkTransactionsClient):
         else:
             logger.info(f"Bulk sync operation succeeded with {success} actions.")
 
-        return f"Successfully added/updated {success} Items. {attempted - success} errors occurred."
+        return f"Successfully added/updated {success} Items. {skipped} skipped (duplicates). {attempted - success} errors occurred."

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/stac_fastapi/core/extensions/catalogs.py

@@ -12,9 +12,15 @@ from starlette.responses import Response
 from typing_extensions import TypedDict
 
 from stac_fastapi.core.models import Catalog
-from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
+from stac_fastapi.sfeos_helpers.database import (
+    search_children_with_pagination_shared,
+    search_collections_by_parent_id_shared,
+    search_sub_catalogs_with_pagination_shared,
+    update_catalog_in_index_shared,
+)
 from stac_fastapi.types import stac as stac_types
 from stac_fastapi.types.core import BaseCoreClient
+from stac_fastapi.types.errors import NotFoundError
 from stac_fastapi.types.extension import ApiExtension
 
 logger = logging.getLogger(__name__)
@@ -106,6 +112,31 @@ class CatalogsExtension(ApiExtension):
             tags=["Catalogs"],
         )
 
+        # Add endpoint for getting sub-catalogs of a catalog
+        self.router.add_api_route(
+            path="/catalogs/{catalog_id}/catalogs",
+            endpoint=self.get_catalog_catalogs,
+            methods=["GET"],
+            response_model=Catalogs,
+            response_class=self.response_class,
+            summary="Get Catalog Sub-Catalogs",
+            description="Get sub-catalogs linked from a specific catalog.",
+            tags=["Catalogs"],
+        )
+
+        # Add endpoint for creating sub-catalogs in a catalog
+        self.router.add_api_route(
+            path="/catalogs/{catalog_id}/catalogs",
+            endpoint=self.create_catalog_catalog,
+            methods=["POST"],
+            response_model=Catalog,
+            response_class=self.response_class,
+            status_code=201,
+            summary="Create Catalog Sub-Catalog",
+            description="Create a new catalog and link it as a sub-catalog of a specific catalog.",
+            tags=["Catalogs"],
+        )
+
         # Add endpoint for creating collections in a catalog
         self.router.add_api_route(
             path="/catalogs/{catalog_id}/collections",
@@ -119,6 +150,18 @@ class CatalogsExtension(ApiExtension):
             tags=["Catalogs"],
         )
 
+        # Add endpoint for updating a catalog
+        self.router.add_api_route(
+            path="/catalogs/{catalog_id}",
+            endpoint=self.update_catalog,
+            methods=["PUT"],
+            response_model=Catalog,
+            response_class=self.response_class,
+            summary="Update Catalog",
+            description="Update an existing STAC catalog.",
+            tags=["Catalogs"],
+        )
+
         # Add endpoint for deleting a catalog
         self.router.add_api_route(
             path="/catalogs/{catalog_id}",
@@ -281,6 +324,55 @@ class CatalogsExtension(ApiExtension):
         # Return the created catalog
         return catalog
 
+    async def update_catalog(
+        self, catalog_id: str, catalog: Catalog, request: Request
+    ) -> Catalog:
+        """Update an existing catalog.
+
+        Args:
+            catalog_id: The ID of the catalog to update.
+            catalog: The updated catalog data.
+            request: Request object.
+
+        Returns:
+            The updated catalog.
+
+        Raises:
+            HTTPException: If the catalog is not found.
+        """
+        try:
+            # Verify the catalog exists
+            existing_catalog_db = await self.client.database.find_catalog(catalog_id)
+
+            # Convert STAC catalog to database format
+            db_catalog = self.client.catalog_serializer.stac_to_db(catalog, request)
+            db_catalog_dict = db_catalog.model_dump()
+            db_catalog_dict["type"] = "Catalog"
+
+            # Preserve parent_ids and other internal fields from the existing catalog
+            if "parent_ids" in existing_catalog_db:
+                db_catalog_dict["parent_ids"] = existing_catalog_db["parent_ids"]
+
+            # Update the catalog in the database (upsert via create_catalog)
+            await self.client.database.create_catalog(db_catalog_dict, refresh=True)
+
+            # Return the updated catalog
+            return catalog
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            error_msg = str(e)
+            if "not found" in error_msg.lower():
+                raise HTTPException(
+                    status_code=404, detail=f"Catalog {catalog_id} not found"
+                )
+            logger.error(f"Error updating catalog {catalog_id}: {e}")
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to update catalog: {str(e)}",
+            )
+
     async def get_catalog(self, catalog_id: str, request: Request) -> Catalog:
         """Get a specific catalog by ID.
 
@@ -365,11 +457,9 @@ class CatalogsExtension(ApiExtension):
             await self.client.database.find_catalog(catalog_id)
 
             # Find all collections with this catalog in parent_ids
-            query_body = {"query": {"term": {"parent_ids": catalog_id}}, "size": 10000}
-            search_result = await self.client.database.client.search(
-                index=COLLECTIONS_INDEX, body=query_body
+            children = await search_collections_by_parent_id_shared(
+                self.client.database.client, catalog_id
             )
-            children = [hit["_source"] for hit in search_result["hits"]["hits"]]
 
             # Safe Unlink: Remove catalog from all children's parent_ids
             # If a child becomes an orphan, adopt it to root
@@ -440,27 +530,15 @@ class CatalogsExtension(ApiExtension):
             # Verify the catalog exists
             await self.client.database.find_catalog(catalog_id)
 
-            # Query collections by parent_ids field using Elasticsearch directly
+            # Query collections by parent_ids field
             # This uses the parent_ids field in the collection mapping to find all
             # collections that have this catalog as a parent
-            query_body = {"query": {"term": {"parent_ids": catalog_id}}}
-
-            # Execute the search to get collection IDs
-            try:
-                search_result = await self.client.database.client.search(
-                    index=COLLECTIONS_INDEX, body=query_body
-                )
-            except Exception as e:
-                logger.error(
-                    f"Error searching for collections with parent {catalog_id}: {e}"
-                )
-                search_result = {"hits": {"hits": []}}
+            collections_data = await search_collections_by_parent_id_shared(
+                self.client.database.client, catalog_id
+            )
 
-            # Extract collection IDs from search results
-            collection_ids = []
-            hits = search_result.get("hits", {}).get("hits", [])
-            for hit in hits:
-                collection_ids.append(hit.get("_id"))
+            # Extract collection IDs from results
+            collection_ids = [coll.get("id") for coll in collections_data]
 
             # Fetch the collections
             collections = []
@@ -524,6 +602,194 @@ class CatalogsExtension(ApiExtension):
                 status_code=404, detail=f"Catalog {catalog_id} not found"
             )
 
+    async def get_catalog_catalogs(
+        self,
+        catalog_id: str,
+        request: Request,
+        limit: int = Query(10, ge=1, le=100),
+        token: Optional[str] = Query(None),
+    ) -> Catalogs:
+        """Get all sub-catalogs of a specific catalog with pagination.
+
+        Args:
+            catalog_id: The ID of the parent catalog.
+            request: Request object.
+            limit: Maximum number of results to return (default: 10, max: 100).
+            token: Pagination token for cursor-based pagination.
+
+        Returns:
+            A Catalogs response containing sub-catalogs with pagination links.
+
+        Raises:
+            HTTPException: If the catalog is not found.
+        """
+        try:
+            # Verify the catalog exists
+            await self.client.database.find_catalog(catalog_id)
+
+            # Search for sub-catalogs with pagination
+            (
+                catalogs_data,
+                total_hits,
+                next_token,
+            ) = await search_sub_catalogs_with_pagination_shared(
+                self.client.database.client, catalog_id, limit, token
+            )
+
+            # Serialize to STAC format
+            catalogs = []
+            for catalog_data in catalogs_data:
+                try:
+                    catalog = self.client.catalog_serializer.db_to_stac(
+                        catalog_data,
+                        request,
+                        extensions=[
+                            type(ext).__name__
+                            for ext in self.client.database.extensions
+                        ],
+                    )
+                    catalogs.append(catalog)
+                except Exception as e:
+                    logger.error(
+                        f"Error serializing catalog {catalog_data.get('id')}: {e}"
+                    )
+                    continue
+
+            # Generate pagination links
+            base_url = str(request.base_url)
+            links = [
+                {"rel": "root", "type": "application/json", "href": base_url},
+                {
+                    "rel": "parent",
+                    "type": "application/json",
+                    "href": f"{base_url}catalogs/{catalog_id}",
+                },
+                {
+                    "rel": "self",
+                    "type": "application/json",
+                    "href": str(request.url),
+                },
+            ]
+
+            # Add next link if more results exist
+            if next_token:
+                query_params = {"limit": limit, "token": next_token}
+                links.append(
+                    {
+                        "rel": "next",
+                        "href": f"{base_url}catalogs/{catalog_id}/catalogs?{urlencode(query_params)}",
+                        "type": "application/json",
+                        "title": "Next page",
+                    }
+                )
+
+            return {
+                "catalogs": catalogs,
+                "links": links,
+                "numberReturned": len(catalogs),
+                "numberMatched": total_hits,
+            }
+
+        except HTTPException:
+            # Re-raise HTTP exceptions as-is
+            raise
+        except Exception as e:
+            logger.error(
+                f"Error retrieving catalogs for catalog {catalog_id}: {e}",
+                exc_info=True,
+            )
+            raise HTTPException(
+                status_code=404, detail=f"Catalog {catalog_id} not found"
+            )
+
+    async def create_catalog_catalog(
+        self, catalog_id: str, catalog: Catalog, request: Request
+    ) -> Catalog:
+        """Create a new catalog or link an existing catalog as a sub-catalog.
+
+        Logic:
+        1. Verifies the parent catalog exists.
+        2. If the sub-catalog already exists: Appends the parent ID to its parent_ids
+           (enabling poly-hierarchy - a catalog can have multiple parents).
+        3. If the sub-catalog is new: Creates it with parent_ids initialized to [catalog_id].
+
+        Args:
+            catalog_id: The ID of the parent catalog.
+            catalog: The catalog to create or link.
+            request: Request object.
+
+        Returns:
+            The created or linked catalog.
+
+        Raises:
+            HTTPException: If the parent catalog is not found or operation fails.
+        """
+        try:
+            # 1. Verify the parent catalog exists
+            await self.client.database.find_catalog(catalog_id)
+
+            # 2. Check if the sub-catalog already exists
+            try:
+                existing_catalog = await self.client.database.find_catalog(catalog.id)
+
+                # --- UPDATE PATH (Existing Catalog) ---
+                # We are linking an existing catalog to a new parent (poly-hierarchy)
+
+                # Ensure parent_ids list exists
+                if "parent_ids" not in existing_catalog:
+                    existing_catalog["parent_ids"] = []
+
+                # Append if not already present
+                if catalog_id not in existing_catalog["parent_ids"]:
+                    existing_catalog["parent_ids"].append(catalog_id)
+
+                    # Persist the update
+                    await update_catalog_in_index_shared(
+                        self.client.database.client, catalog.id, existing_catalog
+                    )
+                    logger.info(
+                        f"Linked existing catalog {catalog.id} to parent {catalog_id}"
+                    )
+
+                # Return the STAC object
+                return self.client.catalog_serializer.db_to_stac(
+                    existing_catalog, request
+                )
+
+            except NotFoundError:
+                # --- CREATE PATH (New Catalog) ---
+                # Catalog does not exist, so we create it
+
+                # Convert STAC catalog to database format
+                db_catalog = self.client.catalog_serializer.stac_to_db(catalog, request)
+
+                # Convert to dict
+                db_catalog_dict = db_catalog.model_dump()
+                db_catalog_dict["type"] = "Catalog"
+
+                # Initialize parent_ids
+                db_catalog_dict["parent_ids"] = [catalog_id]
+
+                # Create in DB
+                await self.client.database.create_catalog(db_catalog_dict, refresh=True)
+                logger.info(
+                    f"Created new catalog {catalog.id} with parent {catalog_id}"
+                )
+
+                return catalog
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(
+                f"Error processing sub-catalog {catalog.id} in parent {catalog_id}: {e}",
+                exc_info=True,
+            )
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to process sub-catalog: {str(e)}",
+            )
+
     async def create_catalog_collection(
         self, catalog_id: str, collection: Collection, request: Request
     ) -> stac_types.Collection:
@@ -792,57 +1058,14 @@ class CatalogsExtension(ApiExtension):
         # 1. Verify the parent catalog exists
         await self.client.database.find_catalog(catalog_id)
 
-        # 2. Build the Search Query
-        # We search the COLLECTIONS_INDEX because it holds both Catalogs and Collections
-
-        # Base filter: Parent match
-        # This finds anything where 'parent_ids' contains this catalog_id
-        filter_queries = [{"term": {"parent_ids": catalog_id}}]
-
-        # Optional filter: Type
-        if type:
-            # If user asks for ?type=Catalog, we only return Catalogs
-            filter_queries.append({"term": {"type": type}})
-
-        # 3. Calculate Pagination (Search After)
-        body = {
-            "query": {"bool": {"filter": filter_queries}},
-            "sort": [{"id": {"order": "asc"}}],  # Stable sort for pagination
-            "size": limit,
-        }
-
-        # Handle search_after token - split by '|' to get all sort values
-        search_after: Optional[List[str]] = None
-        if token:
-            try:
-                # The token should be a pipe-separated string of sort values
-                # e.g., "collection-1"
-                from typing import cast
-
-                search_after_parts = cast(List[str], token.split("|"))
-                # If the number of sort fields doesn't match token parts, ignore the token
-                if len(search_after_parts) != len(body["sort"]):  # type: ignore
-                    search_after = None
-                else:
-                    search_after = search_after_parts
-            except Exception:
-                search_after = None
-
-            if search_after is not None:
-                body["search_after"] = search_after
-
-        # 4. Execute Search
-        search_result = await self.client.database.client.search(
-            index=COLLECTIONS_INDEX, body=body
+        # 2. Search for children with pagination
+        children_data, total, next_token = await search_children_with_pagination_shared(
+            self.client.database.client, catalog_id, limit, token, type
         )
 
-        # 5. Process Results
-        hits = search_result.get("hits", {}).get("hits", [])
-        total = search_result.get("hits", {}).get("total", {}).get("value", 0)
-
+        # 3. Serialize children based on type
         children = []
-        for hit in hits:
-            doc = hit["_source"]
+        for doc in children_data:
             resource_type = doc.get(
                 "type", "Collection"
             )  # Default to Collection if missing
@@ -856,7 +1079,7 @@ class CatalogsExtension(ApiExtension):
 
             children.append(child)
 
-        # 6. Format Response
+        # 4. Format Response
         # The Children extension uses a specific response format
         response = {
             "children": children,
@@ -877,14 +1100,7 @@ class CatalogsExtension(ApiExtension):
             "numberMatched": total,
         }
 
-        # 7. Generate Next Link
-        next_token = None
-        if len(hits) == limit:
-            next_token_values = hits[-1].get("sort")
-            if next_token_values:
-                # Join all sort values with '|' to create the token
-                next_token = "|".join(str(val) for val in next_token_values)
-
+        # 5. Generate Next Link
         if next_token:
             # Get existing query params
             parsed_url = urlparse(str(request.url))

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/stac_fastapi/core/serializers.py

@@ -355,6 +355,9 @@ class CatalogSerializer(Serializer):
         # Avoid modifying the input dict in-place
         catalog = deepcopy(catalog)
 
+        # Remove internal fields (not part of STAC spec)
+        catalog.pop("parent_ids", None)
+
         # Set defaults
         catalog.setdefault("type", "Catalog")
         catalog.setdefault("stac_extensions", [])

{stac_fastapi_core-6.9.0 → stac_fastapi_core-6.10.1}/stac_fastapi/core/version.py

@@ -1,2 +1,2 @@
 """library version."""
-__version__ = "6.9.0"
+__version__ = "6.10.1"