datacosmos 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

datacosmos/stac/item/item_client.py

@@ -0,0 +1,186 @@
+"""STAC Client module for interacting with a STAC (SpatioTemporal Asset Catalog) API.
+
+Provides methods for querying, fetching, creating, updating, and deleting STAC items.
+"""
+
+from typing import Generator, Optional
+
+from pystac import Item
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.exceptions.datacosmos_exception import DatacosmosException
+from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
+from datacosmos.stac.item.models.item_update import ItemUpdate
+from datacosmos.stac.item.models.search_parameters import SearchParameters
+from datacosmos.utils.http_response.check_api_response import check_api_response
+
+
+class ItemClient:
+    """Client for interacting with the STAC API."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Initialize the STACClient with a DatacosmosClient.
+
+        Args:
+            client (DatacosmosClient): The authenticated Datacosmos client instance.
+        """
+        self.client = client
+        self.base_url = client.config.stac.as_domain_url()
+
+    def fetch_item(self, item_id: str, collection_id: str) -> Item:
+        """Fetch a single STAC item by ID.
+
+        Args:
+            item_id (str): The ID of the item to fetch.
+            collection_id (str): The ID of the collection containing the item.
+
+        Returns:
+            Item: The fetched STAC item.
+        """
+        url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item_id}")
+        response = self.client.get(url)
+        check_api_response(response)
+        return Item.from_dict(response.json())
+
+    def fetch_collection_items(
+        self, collection_id: str, parameters: Optional[SearchParameters] = None
+    ) -> Generator[Item, None, None]:
+        """Fetch all items in a collection with optional filtering.
+
+        Args:
+            collection_id (str): The ID of the collection.
+            parameters (Optional[SearchParameters]): Filtering parameters (spatial, temporal, etc.).
+
+        Yields:
+            Item: Parsed STAC item.
+        """
+        if parameters is None:
+            parameters = SearchParameters(collections=[collection_id])
+
+        return self.search_items(parameters)
+
+    def search_items(self, parameters: SearchParameters) -> Generator[Item, None, None]:
+        """Query the STAC catalog using the POST endpoint with filtering and pagination.
+
+        Args:
+            parameters (SearchParameters): The search parameters.
+
+        Yields:
+            Item: Parsed STAC item.
+        """
+        url = self.base_url.with_suffix("/search")
+        body = parameters.model_dump(by_alias=True, exclude_none=True)
+        return self._paginate_items(url, body)
+
+    def create_item(self, collection_id: str, item: Item | DatacosmosItem) -> None:
+        """Create a new STAC item in a specified collection.
+
+        Args:
+            collection_id (str): The ID of the collection where the item will be created.
+            item (Item): The STAC Item to be created.
+
+        Raises:
+            RequestError: If the API returns an error response.
+        """
+        url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
+        item_json: dict = item.to_dict()
+
+        response = self.client.post(url, json=item_json)
+        check_api_response(response)
+
+    def update_item(
+        self, item_id: str, collection_id: str, update_data: ItemUpdate
+    ) -> None:
+        """Partially update an existing STAC item.
+
+        Args:
+            item_id (str): The ID of the item to update.
+            collection_id (str): The ID of the collection containing the item.
+            update_data (ItemUpdate): The structured update payload.
+        """
+        url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item_id}")
+
+        update_payload = update_data.model_dump(by_alias=True, exclude_none=True)
+
+        if "assets" in update_payload:
+            update_payload["assets"] = {
+                key: asset.to_dict() for key, asset in update_payload["assets"].items()
+            }
+        if "links" in update_payload:
+            update_payload["links"] = [
+                link.to_dict() for link in update_payload["links"]
+            ]
+
+        response = self.client.patch(url, json=update_payload)
+        check_api_response(response)
+
+    def delete_item(self, item_id: str, collection_id: str) -> None:
+        """Delete a STAC item by its ID.
+
+        Args:
+            item_id (str): The ID of the item to delete.
+            collection_id (str): The ID of the collection containing the item.
+
+        Raises:
+            OCError: If the item is not found or deletion is forbidden.
+        """
+        url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item_id}")
+        response = self.client.delete(url)
+        check_api_response(response)
+
+    def _paginate_items(self, url: str, body: dict) -> Generator[Item, None, None]:
+        """Handle pagination for the STAC search POST endpoint.
+
+        Fetches items one page at a time using the 'next' link.
+
+        Args:
+            url (str): The base URL for the search endpoint.
+            body (dict): The request body containing search parameters.
+
+        Yields:
+            Item: Parsed STAC item.
+        """
+        params = {"limit": body.get("limit", 10)}
+
+        while True:
+            response = self.client.post(url, json=body, params=params)
+            check_api_response(response)
+            data = response.json()
+
+            yield from (Item.from_dict(feature) for feature in data.get("features", []))
+
+            next_href = self._get_next_link(data)
+            if not next_href:
+                break
+
+            token = self._extract_pagination_token(next_href)
+            if not token:
+                break
+            params["cursor"] = token
+
+    def _get_next_link(self, data: dict) -> Optional[str]:
+        """Extract the next page link from the response."""
+        next_link = next(
+            (link for link in data.get("links", []) if link.get("rel") == "next"), None
+        )
+        return next_link.get("href", "") if next_link else None
+
+    def _extract_pagination_token(self, next_href: str) -> Optional[str]:
+        """Extract the pagination token from the next link URL.
+
+        Args:
+            next_href (str): The next page URL.
+
+        Returns:
+            Optional[str]: The extracted token, or None if parsing fails.
+
+        Raises:
+            DatacosmosException: If pagination token extraction fails.
+        """
+        try:
+            return next_href.split("?")[1].split("=")[-1]
+        except (IndexError, AttributeError) as e:
+            raise DatacosmosException(
+                f"Failed to parse pagination token from {next_href}",
+                response=e.response,
+            ) from e

datacosmos/stac/item/models/__init__.py

@@ -0,0 +1 @@
+"""Models for the Item Client."""

datacosmos/stac/item/models/asset.py

@@ -0,0 +1,23 @@
+"""Model representing a datacosmos item asset."""
+
+from pydantic import BaseModel, Field
+
+from datacosmos.stac.item.models.eo_band import EoBand
+from datacosmos.stac.item.models.raster_band import RasterBand
+
+
+class Asset(BaseModel):
+    """Model representing a datacosmos item asset."""
+
+    href: str
+    title: str
+    description: str
+    type: str
+    roles: list[str] | None
+    eo_bands: list[EoBand] | None = Field(default=None, alias="eo:bands")
+    raster_bands: list[RasterBand] | None = Field(default=None, alias="raster:bands")
+
+    class Config:
+        """Pydantic configuration."""
+
+        populate_by_name = True

datacosmos/stac/item/models/datacosmos_item.py

@@ -0,0 +1,55 @@
+"""Model representing a datacosmos item."""
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+from datacosmos.stac.enums.level import Level
+from datacosmos.stac.item.models.asset import Asset
+
+
+class DatacosmosItem(BaseModel):
+    """Model representing a datacosmos item."""
+
+    id: str
+    type: str
+    stac_version: str
+    stac_extensions: list | None
+    geometry: dict
+    properties: dict
+    links: list
+    assets: dict[str, Asset]
+    collection: str
+    bbox: tuple[float, float, float, float]
+
+    def get_property(self, key: str) -> str | None:
+        """Get a property value from the Datacosmos item."""
+        return self.properties.get(key)
+
+    def get_asset(self, key: str) -> Asset | None:
+        """Get an asset from the Datacosmos item."""
+        return self.assets.get(key)
+
+    @property
+    def datetime(self) -> datetime:
+        """Get the datetime of the Datacosmos item."""
+        return datetime.strptime(self.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
+
+    @property
+    def level(self) -> Level:
+        """Get the processing level of the Datacosmos item."""
+        return Level(self.properties["processing:level"].lower())
+
+    @property
+    def sat_int_designator(self) -> str:
+        """Get the satellite international designator of the Datacosmos item."""
+        property = self.get_property("sat:platform_international_designator")
+        if property is None:
+            raise ValueError(
+                "sat:platform_international_designator is missing in STAC item"
+            )
+        return property
+
+    def to_dict(self) -> dict:
+        """Converts the DatacosmosItem instance to a dictionary."""
+        return self.model_dump()

datacosmos/stac/item/models/eo_band.py

@@ -0,0 +1,15 @@
+"""Model representing an EO band."""
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class EoBand(BaseModel):
+    """Model representing an EO band."""
+
+    name: str
+    common_name: str
+    center_wavelength: float
+    full_width_half_max: float
+    solar_illumination: Optional[float] = None

datacosmos/stac/item/models/item_update.py

@@ -0,0 +1,57 @@
+"""Model representing a partial update for a STAC item."""
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, model_validator
+from pystac import Asset, Link
+
+
+class ItemUpdate(BaseModel):
+    """Model representing a partial update for a STAC item."""
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    stac_extensions: Optional[list[str]] = None
+    geometry: Optional[dict[str, Any]] = None
+    bbox: Optional[list[float]] = Field(
+        None, min_items=4, max_items=4
+    )  # Must be [minX, minY, maxX, maxY]
+    properties: Optional[dict[str, Any]] = None
+    assets: Optional[dict[str, Asset]] = None
+    links: Optional[list[Link]] = None
+
+    def set_geometry(self, geom_type: str, coordinates: list[Any]) -> None:
+        """Set the geometry manually without using shapely.
+
+        Args:
+            geom_type (str): The type of geometry (e.g., 'Point', 'Polygon').
+            coordinates (list[Any]): The coordinates defining the geometry.
+        """
+        self.geometry = {"type": geom_type, "coordinates": coordinates}
+
+    @staticmethod
+    def has_valid_datetime(properties: dict[str, Any]) -> bool:
+        """Check if 'datetime' is present and not None."""
+        return properties.get("datetime") is not None
+
+    @staticmethod
+    def has_valid_datetime_range(properties: dict[str, Any]) -> bool:
+        """Check if both 'start_datetime' and 'end_datetime' are present and not None."""
+        return all(
+            properties.get(key) is not None
+            for key in ["start_datetime", "end_datetime"]
+        )
+
+    @model_validator(mode="before")
+    def validate_datetime_fields(cls, values):
+        """Ensure at least one of 'datetime' or 'start_datetime'/'end_datetime' exists."""
+        properties = values.get("properties", {})
+
+        if not cls.has_valid_datetime(properties) and not cls.has_valid_datetime_range(
+            properties
+        ):
+            raise ValueError(
+                "Either 'datetime' or both 'start_datetime' and 'end_datetime' must be provided."
+            )
+
+        return values

datacosmos/stac/item/models/raster_band.py

@@ -0,0 +1,17 @@
+"""Model representing a raster band."""
+
+from pydantic import BaseModel, Field
+
+
+class RasterBand(BaseModel):
+    """Model representing a raster band."""
+
+    gain: float = Field(alias="scale")
+    bias: float = Field(alias="offset")
+    nodata: int
+    unit: str
+
+    class Config:
+        """Pydantic configuration."""
+
+        populate_by_name = True

datacosmos/stac/item/models/search_parameters.py

@@ -0,0 +1,58 @@
+"""Module defining the SearchParameters model for STAC API queries, encapsulating filtering criteria.
+
+It includes spatial, temporal, and property-based filters for querying STAC items efficiently.
+"""
+
+from typing import Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class SearchParameters(BaseModel):
+    """Encapsulates the parameters for the STAC search API with validation."""
+
+    bbox: Optional[list[float]] = Field(
+        None,
+        description="Bounding box filter [minX, minY, maxX, maxY]. Optional six values for 3D bounding box.",
+        example=[-180.0, -90.0, 180.0, 90.0],
+    )
+    datetime_range: Optional[str] = Field(
+        None,
+        alias="datetime",
+        description=(
+            "Temporal filter, either a single RFC 3339 datetime or an interval. "
+            'Example: "2025-01-01T00:00:00Z/.."'
+        ),
+    )
+    intersects: Optional[dict] = Field(
+        None, description="GeoJSON geometry filter, e.g., a Polygon or Point."
+    )
+    ids: Optional[list[str]] = Field(
+        None,
+        description="Array of item IDs to filter by.",
+        example=["item1", "item2"],
+    )
+    collections: Optional[list[str]] = Field(
+        None,
+        description="Array of collection IDs to filter by.",
+        example=["collection1", "collection2"],
+    )
+    limit: Optional[int] = Field(
+        None,
+        ge=1,
+        le=10000,
+        description="Maximum number of items per page. Default: 10, Max: 10000.",
+        example=10,
+    )
+    query: Optional[dict[str, dict[str, Union[str, int, float]]]] = Field(
+        None,
+        description="Additional property filters, e.g., { 'cloud_coverage': { 'lt': 10 } }.",
+    )
+
+    @model_validator(mode="before")
+    def validate_bbox(cls, values):
+        """Validate that the `bbox` field contains either 4 or 6 values."""
+        bbox = values.get("bbox")
+        if bbox and len(bbox) not in {4, 6}:
+            raise ValueError("bbox must contain 4 or 6 values.")
+        return values

datacosmos/stac/stac_client.py

@@ -0,0 +1,12 @@
+"""Unified interface for STAC API, combining Item & Collection operations."""
+
+from datacosmos.stac.collection.collection_client import CollectionClient
+from datacosmos.stac.item.item_client import ItemClient
+
+
+class STACClient(ItemClient, CollectionClient):
+    """Unified interface for STAC API, combining Item & Collection operations."""
+
+    def __init__(self, client):
+        """Initialize the STACClient with a DatacosmosClient."""
+        super().__init__(client)

datacosmos/uploader/__init__.py

@@ -0,0 +1 @@
+"""Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""

datacosmos/uploader/dataclasses/__init__.py

@@ -0,0 +1 @@
+"""Dataclasses for the uploader module."""

datacosmos/uploader/dataclasses/upload_path.py

@@ -0,0 +1,93 @@
+"""Dataclass for retrieving the upload path of a file."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+
+from datacosmos.stac.enums.level import Level
+from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
+from datacosmos.utils.missions import get_mission_id
+
+logger = structlog.get_logger()
+
+
+@dataclass
+class UploadPath:
+    """Dataclass for retrieving the upload path of a file."""
+
+    mission: str
+    level: Level
+    day: int
+    month: int
+    year: int
+    id: str
+    path: str
+
+    def __str__(self):
+        """Return a human-readable string representation of the Path."""
+        path = f"full/{self.mission.lower()}/{self.level.value.lower()}/{self.year:02}/{self.month:02}/{self.day:02}/{self.id}/{self.path}"
+        return path.removesuffix("/")
+
+    @classmethod
+    def from_item_path(
+        cls, item: DatacosmosItem, mission: str, item_path: str
+    ) -> "Path":
+        """Create a Path instance from a DatacosmosItem and a path."""
+        for asset in item.assets.values():
+            if mission == "":
+                mission = cls._get_mission_name(asset.href)
+            else:
+                break
+        dt = datetime.strptime(item.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
+        path = UploadPath(
+            mission=mission,
+            level=Level(item.properties["processing:level"].lower()),
+            day=dt.day,
+            month=dt.month,
+            year=dt.year,
+            id=item.id,
+            path=item_path,
+        )
+        return cls(**path.__dict__)
+
+    @classmethod
+    def from_path(cls, path: str) -> "Path":
+        """Create a Path instance from a string path."""
+        parts = path.split("/")
+        if len(parts) < 7:
+            raise ValueError(f"Invalid path {path}")
+        return cls(
+            mission=parts[0],
+            level=Level(parts[1]),
+            day=int(parts[4]),
+            month=int(parts[3]),
+            year=int(parts[2]),
+            id=parts[5],
+            path="/".join(parts[6:]),
+        )
+
+    @classmethod
+    def _get_mission_name(cls, href: str) -> str:
+        mission = ""
+        # bruteforce mission name from asset path
+        # traverse the path and check if any part is a mission name (generates a mission id)
+        href_parts = href.split("/")
+        for idx, part in enumerate(href_parts):
+            try:
+                # when an id is found, then the mission name is valid
+                get_mission_id(
+                    part, "test"
+                )  # using test as it is more wide and anything on prod should exists on test
+            except KeyError:
+                continue
+            # validate the mission name by checking if the path is correct
+            # using the same logic as the __str__ method
+            mission = part.lower()
+            h = "/".join(["full", *href_parts[idx:]])
+            p = UploadPath.from_path("/".join([mission, *href_parts[idx + 1 :]]))
+            if str(p) != h:
+                raise ValueError(f"Could not find mission name in asset path {href}")
+            break
+        return mission

datacosmos/uploader/datacosmos_uploader.py

@@ -0,0 +1,106 @@
+"""Module for uploading files to Datacosmos cloud storage and registering STAC items."""
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from pydantic import TypeAdapter
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.stac.item.item_client import ItemClient
+from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
+from datacosmos.uploader.dataclasses.upload_path import UploadPath
+from datacosmos.utils.missions import get_mission_name
+
+
+class DatacosmosUploader:
+    """Handles uploading files to Datacosmos storage and registering STAC items."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Initialize the uploader with DatacosmosClient."""
+        mission_id = client.config.mission_id
+        environment = client.config.environment
+
+        self.datacosmos_client = client
+        self.item_client = ItemClient(client)
+        self.mission_name = (
+            get_mission_name(mission_id, environment) if mission_id != 0 else ""
+        )
+        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
+
+    def upload_and_register_item(self, item_json_file_path: str) -> None:
+        """Uploads files to Datacosmos storage and registers a STAC item.
+
+        Args:
+            item_json_file_path (str): Path to the STAC item JSON file.
+        """
+        item = self._load_item(item_json_file_path)
+        collection_id, item_id = item.collection, item.id
+        dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
+
+        self._delete_existing_item(collection_id, item_id)
+        upload_path = self._get_upload_path(item)
+        self.upload_from_folder(dirname, upload_path)
+
+        self._update_item_assets(item)
+
+        self.item_client.create_item(collection_id, item)
+
+    def upload_file(self, src: str, dst: str) -> None:
+        """Uploads a single file to the specified destination path."""
+        url = self.base_url.with_suffix(str(dst))
+
+        with open(src, "rb") as f:
+            response = self.datacosmos_client.put(url, data=f)
+        response.raise_for_status()
+
+    def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
+        """Uploads all files from a folder to the destination path in parallel."""
+        if Path(dst.path).is_file():
+            raise ValueError(f"Destination path should not be a file path {dst}")
+
+        if Path(src).is_file():
+            raise ValueError(f"Source path should not be a file path {src}")
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = []
+            for file in Path(src).rglob("*"):
+                if file.is_file():
+                    dst = UploadPath(
+                        mission=dst.mission,
+                        level=dst.level,
+                        day=dst.day,
+                        month=dst.month,
+                        year=dst.year,
+                        id=dst.id,
+                        path=str(file.relative_to(src)),
+                    )
+                    futures.append(executor.submit(self.upload_file, str(file), dst))
+            for future in futures:
+                future.result()
+
+    @staticmethod
+    def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        """Loads and validates the STAC item from a JSON file."""
+        with open(item_json_file_path, "rb") as file:
+            data = file.read().decode("utf-8")
+        return TypeAdapter(DatacosmosItem).validate_json(data)
+
+    def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
+        """Deletes an existing item if it already exists."""
+        try:
+            self.item_client.delete_item(item_id, collection_id)
+        except Exception:  # nosec
+            pass  # Ignore if item doesn't exist
+
+    def _get_upload_path(self, item: DatacosmosItem) -> str:
+        """Constructs the storage upload path based on the item and mission name."""
+        return UploadPath.from_item_path(item, self.mission_name, "")
+
+    def _update_item_assets(self, item: DatacosmosItem) -> None:
+        """Updates the item's assets with uploaded file URLs."""
+        for asset in item.assets.values():
+            try:
+                url = self.base_url
+                asset.href = url.with_base(asset.href)  # type: ignore
+            except ValueError:
+                pass

datacosmos/utils/__init__.py

@@ -0,0 +1 @@
+"""Http response and url utils for datacosmos."""

datacosmos/utils/constants.py

@@ -0,0 +1,16 @@
+"""Package for storing constants."""
+
+TEST_MISSION_NAMES = {
+    55: "MENUT",
+    56: "PHISAT-2",
+    57: "HAMMER",
+    63: "MANTIS",
+    64: "PLATERO",
+}
+PROD_MISSION_NAMES = {
+    23: "MENUT",
+    29: "MANTIS",
+    35: "PHISAT-2",
+    37: "PLATERO",
+    48: "HAMMER",
+}

datacosmos/utils/http_response/__init__.py

@@ -0,0 +1 @@
+"""Validates an API response."""