datacosmos 0.0.8__tar.gz → 0.0.9__tar.gz

{datacosmos-0.0.8 → datacosmos-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.8
+Version: 0.0.9
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.8 → datacosmos-0.0.9}/README.md

@@ -83,8 +83,10 @@ client = DatacosmosClient(config=config)
 | `datacosmos_cloud_storage.host`     | `app.open-cosmos.com`                   | YAML / ENV     |
 | `datacosmos_cloud_storage.port`     | `443`                                   | YAML / ENV     |
 | `datacosmos_cloud_storage.path`     | `/api/data/v0/storage`                 | YAML / ENV     |
-| `mission_id`                | `0`                                             | YAML / ENV     |
-| `environment`               | `test`                                          | YAML / ENV     |
+| `datacosmos_public_cloud_storage.protocol` | `https`                                 | YAML / ENV     |
+| `datacosmos_public_cloud_storage.host`     | `app.open-cosmos.com`                   | YAML / ENV     |
+| `datacosmos_public_cloud_storage.port`     | `443`                                   | YAML / ENV     |
+| `datacosmos_public_cloud_storage.path`     | `/api/data/v0/storage`                 | YAML / ENV     |
 
 ## STAC Client
 
@@ -163,7 +165,41 @@ stac_item.add_asset(
     )
 )
 
-stac_client.create_item(collection_id="example-collection", item=stac_item)
+# this will raise an error if the stac item already exists. Use the add_item method in that case
+stac_client.create_item(item=stac_item)
+```
+
+#### 4. **Add a STAC Item**
+```python
+from pystac import Item, Asset
+from datetime import datetime
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.stac.stac_client import STACClient
+
+client = DatacosmosClient()
+stac_client = STACClient(client)
+
+stac_item = Item(
+    id="new-item",
+    geometry={"type": "Point", "coordinates": [102.0, 0.5]},
+    bbox=[101.0, 0.0, 103.0, 1.0],
+    datetime=datetime.utcnow(),
+    properties={"datetime": datetime.utcnow(), "processing:level": "example-processing-level"},
+    collection="example-collection"
+)
+
+stac_item.add_asset(
+    "image",
+    Asset(
+        href="https://example.com/sample-image.tiff",
+        media_type="image/tiff",
+        roles=["data"],
+        title="Sample Image"
+    )
+)
+
+stac_client.add_item(item=stac_item)
 ```
 
 #### 4. **Update an Existing STAC Item**
@@ -298,48 +334,44 @@ stac_client.delete_collection("test-collection")
 
 ## Uploading Files and Registering STAC Items
 
-You can use the `DatacosmosUploader` class to upload files to the DataCosmos cloud storage and register a STAC item. The `upload_and_register_item` method will take care of both uploading files and creating the STAC item.
+You can use the `STACClient` class to upload files to the DataCosmos cloud storage and register a STAC item.
 
-### **Upload Files and Register STAC Item**
-
-1. Make sure you have a directory with the same name as your STAC item JSON file (this directory should contain the files you want to upload).
-2. Call the `upload_and_register_item` method, providing the path to the STAC item JSON file.
+### **Upload and add STAC Item**
 
 ```python
-from datacosmos.datacosmos_client import DatacosmosClient
-from datacosmos.uploader.datacosmos_uploader import DatacosmosUploader
+from pystac import Item, Asset
 
-# Initialize the client with the configuration
-client = DatacosmosClient(config=config)
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.stac.stac_client import STACClient
 
-# Create the uploader instance
-uploader = DatacosmosUploader(client)
+client = DatacosmosClient()
 
-# Path to your STAC item JSON file
-item_json_file_path = "/home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814.json"
+stac_client = STACClient(client)
 
-# Upload the item and its assets, and register it in the STAC API
-uploader.upload_and_register_item(item_json_file_path)
-```
+stac_item = Item(
+    id="new-item",
+    geometry={"type": "Point", "coordinates": [102.0, 0.5]},
+    bbox=[101.0, 0.0, 103.0, 1.0],
+    datetime=datetime.utcnow(),
+    properties={"datetime": datetime.utcnow(), "processing:level": "example-processing-level"},
+    collection="example-collection"
+)
 
-### **Folder Structure**
+stac_item.add_asset(
+    "image",
+    Asset(
+        href="https://example.com/sample-image.tiff",
+        media_type="image/tiff",
+        roles=["data"],
+        title="Sample Image"
+    )
+)
 
-For the `upload_and_register_item` method to work correctly, ensure that the directory structure matches the name of the STAC item JSON file. For example:
+assets_path = "path/to/assets"
 
+# Upload the item and its assets, and register it in the STAC API
+stac_client.upload_item(stac_item, assets_path)
 ```
-/home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814.json
-/home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814/
-    ├── asset1.tiff
-    ├── asset2.tiff
-    └── ...
-```
-
-The folder `MENUT_L1A_000001943_20250304134812_20250304134821_49435814` should contain the assets (files) for upload.
-
-The `upload_and_register_item` method will:
-1. Delete any existing item with the same ID (if it exists).
-2. Upload the assets in the folder to DataCosmos cloud storage.
-3. Register the item in the STAC API.
 
 ## Error Handling
 

{datacosmos-0.0.8 → datacosmos-0.0.9}/datacosmos/config/config.py

@@ -29,6 +29,7 @@ class Config(BaseSettings):
     authentication: Optional[AuthenticationConfig] = None
     stac: Optional[URL] = None
     datacosmos_cloud_storage: Optional[URL] = None
+    datacosmos_public_cloud_storage: Optional[URL] = None
 
     DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m"
     DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token"
@@ -86,12 +87,18 @@ class Config(BaseSettings):
             path=os.getenv("DC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
         )
 
+        datacosmos_public_cloud_storage_config = URL(
+            protocol=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PROTOCOL", "https"),
+            host=os.getenv("DC_PUBLIC_CLOUD_STORAGE_HOST", "app.open-cosmos.com"),
+            port=int(os.getenv("DC_PUBLIC_CLOUD_STORAGE_PORT", "443")),
+            path=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
+        )
+
         return cls(
             authentication=authentication_config,
             stac=stac_config,
             datacosmos_cloud_storage=datacosmos_cloud_storage_config,
-            mission_id=int(os.getenv("MISSION_ID", "0")),
-            environment=os.getenv("ENVIRONMENT", "test"),
+            datacosmos_public_cloud_storage=datacosmos_public_cloud_storage_config,
         )
 
     @field_validator("authentication", mode="after")
@@ -191,3 +198,25 @@ class Config(BaseSettings):
                 path="/api/data/v0/storage",
             )
         return datacosmos_cloud_storage_config
+
+    @field_validator("datacosmos_public_cloud_storage", mode="before")
+    @classmethod
+    def validate_datacosmos_public_cloud_storage(
+        cls, datacosmos_public_cloud_storage_config: Optional[URL]
+    ) -> URL:
+        """Ensure datacosmos cloud storage configuration has a default if not explicitly set.
+
+        Args:
+            datacosmos_public_cloud_storage_config (Optional[URL]): The datacosmos public cloud storage config to validate.
+
+        Returns:
+            URL: The validated datacosmos public cloud storage configuration.
+        """
+        if datacosmos_public_cloud_storage_config is None:
+            return URL(
+                protocol="https",
+                host="app.open-cosmos.com",
+                port=443,
+                path="/api/data/v0/storage",
+            )
+        return datacosmos_public_cloud_storage_config

{datacosmos-0.0.8 → datacosmos-0.0.9}/datacosmos/stac/item/item_client.py

@@ -62,21 +62,60 @@ class ItemClient:
             body = body | {"collections": parameters.collections}
         return self._paginate_items(url, body)
 
-    def create_item(self, collection_id: str, item: Item | DatacosmosItem) -> None:
-        """Create a new STAC item in a specified collection.
+    def create_item(self, item: Item | DatacosmosItem) -> None:
+        """Create a new STAC item in its own collection.
+
+        The collection ID is inferred from the item.
 
         Args:
-            collection_id (str): The ID of the collection where the item will be created.
-            item (Item): The STAC Item to be created.
+            item (Item | DatacosmosItem): The STAC item to be created.
 
         Raises:
+            ValueError: If the item has no collection set.
             RequestError: If the API returns an error response.
         """
+        if isinstance(item, Item):
+            collection_id = item.collection_id or (
+                item.get_collection().id if item.get_collection() else None
+            )
+        else:
+            collection_id = item.collection
+
+        if not collection_id:
+            raise ValueError("Cannot create item: no collection_id found on item")
+
         url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
         item_json: dict = item.to_dict()
         response = self.client.post(url, json=item_json)
         check_api_response(response)
 
+    def add_item(self, item: Item | DatacosmosItem) -> None:
+        """Adds item to catalog.
+
+        The collection ID is inferred from the item.
+
+        Args:
+            item (Item | DatacosmosItem): The STAC item to be created.
+
+        Raises:
+            ValueError: If the item has no collection set.
+            RequestError: If the API returns an error response.
+        """
+        if isinstance(item, Item):
+            collection_id = item.collection_id or (
+                item.get_collection().id if item.get_collection() else None
+            )
+        else:
+            collection_id = item.collection
+
+        if not collection_id:
+            raise ValueError("Cannot create item: no collection_id found on item")
+
+        url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item.id}")
+        item_json: dict = item.to_dict()
+        response = self.client.put(url, json=item_json)
+        check_api_response(response)
+
     def update_item(
         self, item_id: str, collection_id: str, update_data: ItemUpdate
     ) -> None:

{datacosmos-0.0.8 → datacosmos-0.0.9}/datacosmos/stac/stac_client.py

@@ -2,11 +2,14 @@
 
 from datacosmos.stac.collection.collection_client import CollectionClient
 from datacosmos.stac.item.item_client import ItemClient
+from datacosmos.stac.storage.storage_client import StorageClient
 
 
-class STACClient(ItemClient, CollectionClient):
+class STACClient(ItemClient, CollectionClient, StorageClient):
     """Unified interface for STAC API, combining Item & Collection operations."""
 
     def __init__(self, client):
         """Initialize the STACClient with a DatacosmosClient."""
-        super().__init__(client)
+        ItemClient.__init__(self, client)
+        CollectionClient.__init__(self, client)
+        StorageClient.__init__(self, client)

datacosmos-0.0.9/datacosmos/stac/storage/__init__.py

@@ -0,0 +1,5 @@
+"""Facade module for all storage-related operations."""
+
+from datacosmos.stac.storage.storage_client import StorageClient
+
+__all__ = ["StorageClient"]

datacosmos-0.0.9/datacosmos/stac/storage/storage_base.py

@@ -0,0 +1,40 @@
+"""Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+import mimetypes
+from concurrent.futures import ThreadPoolExecutor, wait
+
+from datacosmos.datacosmos_client import DatacosmosClient
+
+
+class StorageBase:
+    """Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Base class providing common storage helpers (threading, MIME guess, futures)."""
+        self.client = client
+        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
+
+    def _guess_mime(self, src: str) -> str:
+        mime, _ = mimetypes.guess_type(src)
+        return mime or "application/octet-stream"
+
+    def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
+        """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
+
+        `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
+        """
+        futures = []
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            for args in fn_args:
+                futures.append(executor.submit(fn, *args))
+        done, not_done = wait(futures, timeout=timeout)
+        errors = []
+        for future in done:
+            try:
+                future.result()
+            except Exception as e:
+                errors.append(e)
+        for future in not_done:
+            future.cancel()
+        if errors:
+            raise errors[0]

datacosmos-0.0.9/datacosmos/stac/storage/storage_client.py

@@ -0,0 +1,31 @@
+"""Generic StorageClient for all storage operations (upload, download, etc.)."""
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
+from datacosmos.stac.storage.uploader import Uploader
+
+
+class StorageClient:
+    """Generic StorageClient for all storage operations (upload, download, etc.)."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Generic StorageClient for all storage operations (upload, download, etc.)."""
+        self.client = client
+        self.uploader = Uploader(client)
+
+    def upload_item(
+        self,
+        item: DatacosmosItem,
+        assets_path: str | None = None,
+        included_assets: list[str] | bool = True,
+        max_workers: int = 4,
+        time_out: float = 60 * 60 * 1,
+    ) -> DatacosmosItem:
+        """Proxy to Uploader.upload_item, without needing to pass client each call."""
+        return self.uploader.upload_item(
+            item=item,
+            assets_path=assets_path,
+            included_assets=included_assets,
+            max_workers=max_workers,
+            time_out=time_out,
+        )

datacosmos-0.0.9/datacosmos/stac/storage/uploader.py

@@ -0,0 +1,98 @@
+"""Handles uploading files to Datacosmos storage and registering STAC items."""
+
+from pathlib import Path
+
+from pydantic import TypeAdapter
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.stac.item.item_client import ItemClient
+from datacosmos.stac.item.models.asset import Asset
+from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
+from datacosmos.stac.storage.dataclasses.upload_path import UploadPath
+from datacosmos.stac.storage.storage_base import StorageBase
+
+
+class Uploader(StorageBase):
+    """Handles uploading files to Datacosmos storage and registering STAC items."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Handles uploading files to Datacosmos storage and registering STAC items."""
+        super().__init__(client)
+        self.item_client = ItemClient(client)
+
+    def upload_item(
+        self,
+        item: DatacosmosItem,
+        assets_path: str | None = None,
+        included_assets: list[str] | bool = True,
+        max_workers: int = 4,
+        time_out: float = 60 * 60 * 1,
+    ) -> DatacosmosItem:
+        """Upload a STAC item and its assets to Datacosmos."""
+        if not assets_path and not isinstance(item, str):
+            raise ValueError(
+                "assets_path must be provided if item is not the path to an item file."
+            )
+
+        if isinstance(item, str):
+            item_filename = item
+            item = self._load_item(item_filename)
+            if not assets_path:
+                assets_path = str(Path(item_filename).parent)
+
+        assets_path = assets_path or str(Path.cwd())
+
+        upload_assets = (
+            included_assets
+            if isinstance(included_assets, list)
+            else item.assets.keys()
+            if included_assets is True
+            else []
+        )
+
+        jobs = [(item, asset_key, assets_path) for asset_key in upload_assets]
+
+        self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
+
+        self.item_client.add_item(item)
+
+        return item
+
+    def upload_from_file(
+        self, src: str, dst: str, mime_type: str | None = None
+    ) -> None:
+        """Uploads a single file to the specified destination path."""
+        url = self.base_url.with_suffix(dst)
+        mime = mime_type or self._guess_mime(src)
+        headers = {"Content-Type": mime}
+        with open(src, "rb") as f:
+            response = self.client.put(url, data=f, headers=headers)
+        response.raise_for_status()
+
+    @staticmethod
+    def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        with open(item_json_file_path, "rb") as file:
+            data = file.read().decode("utf-8")
+        return TypeAdapter(DatacosmosItem).validate_json(data)
+
+    def _upload_asset(
+        self, item: DatacosmosItem, asset_key: str, assets_path: str
+    ) -> None:
+        asset = item.assets[asset_key]
+        upload_path = UploadPath.from_item_path(item, "", Path(asset.href).name)
+        local_src = Path(assets_path) / asset.href
+        if local_src.exists():
+            src = str(local_src)
+            asset.href = f"file:///{upload_path}"
+        else:
+            src = str(Path(assets_path) / Path(asset.href).name)
+        self._update_asset_href(asset)
+        self.upload_from_file(src, str(upload_path), mime_type=asset.type)
+
+    def _update_asset_href(self, asset: Asset) -> None:
+        try:
+            url = self.client.config.datacosmos_public_cloud_storage.as_domain_url()
+            new_href = url.with_base(asset.href)  # type: ignore
+            asset.href = str(new_href)
+        except ValueError:
+            pass

{datacosmos-0.0.8 → datacosmos-0.0.9}/datacosmos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.8
+Version: 0.0.9
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.8 → datacosmos-0.0.9}/datacosmos.egg-info/SOURCES.txt

@@ -39,10 +39,12 @@ datacosmos/stac/item/models/datacosmos_item.py
 datacosmos/stac/item/models/eo_band.py
 datacosmos/stac/item/models/item_update.py
 datacosmos/stac/item/models/raster_band.py
-datacosmos/uploader/__init__.py
-datacosmos/uploader/datacosmos_uploader.py
-datacosmos/uploader/dataclasses/__init__.py
-datacosmos/uploader/dataclasses/upload_path.py
+datacosmos/stac/storage/__init__.py
+datacosmos/stac/storage/storage_base.py
+datacosmos/stac/storage/storage_client.py
+datacosmos/stac/storage/uploader.py
+datacosmos/stac/storage/dataclasses/__init__.py
+datacosmos/stac/storage/dataclasses/upload_path.py
 datacosmos/utils/__init__.py
 datacosmos/utils/url.py
 datacosmos/utils/http_response/__init__.py

{datacosmos-0.0.8 → datacosmos-0.0.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datacosmos"
-version = "0.0.8"
+version = "0.0.9"
 authors = [
     { name="Open Cosmos", email="support@open-cosmos.com" },
 ]

datacosmos-0.0.8/datacosmos/uploader/__init__.py

@@ -1 +0,0 @@
-"""Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""

datacosmos-0.0.8/datacosmos/uploader/datacosmos_uploader.py

@@ -1,101 +0,0 @@
-"""Module for uploading files to Datacosmos cloud storage and registering STAC items."""
-
-from concurrent.futures import ThreadPoolExecutor
-from pathlib import Path
-
-from pydantic import TypeAdapter
-
-from datacosmos.datacosmos_client import DatacosmosClient
-from datacosmos.stac.item.item_client import ItemClient
-from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
-from datacosmos.uploader.dataclasses.upload_path import UploadPath
-
-
-class DatacosmosUploader:
-    """Handles uploading files to Datacosmos storage and registering STAC items."""
-
-    def __init__(self, client: DatacosmosClient):
-        """Initialize the uploader with DatacosmosClient."""
-        self.environment = client.config.environment
-
-        self.datacosmos_client = client
-        self.item_client = ItemClient(client)
-        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
-
-    def upload_and_register_item(self, item_json_file_path: str) -> None:
-        """Uploads files to Datacosmos storage and registers a STAC item.
-
-        Args:
-            item_json_file_path (str): Path to the STAC item JSON file.
-        """
-        item = self._load_item(item_json_file_path)
-        collection_id, item_id = item.collection, item.id
-        dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
-
-        self._delete_existing_item(collection_id, item_id)
-        upload_path = self._get_upload_path(item)
-        self.upload_from_folder(dirname, upload_path)
-
-        self._update_item_assets(item)
-
-        self.item_client.create_item(collection_id, item)
-
-    def upload_file(self, src: str, dst: str) -> None:
-        """Uploads a single file to the specified destination path."""
-        url = self.base_url.with_suffix(dst)
-
-        with open(src, "rb") as f:
-            response = self.datacosmos_client.put(url, data=f)
-        response.raise_for_status()
-
-    def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
-        """Uploads all files from a folder to the destination path in parallel."""
-        if Path(dst.path).is_file():
-            raise ValueError(f"Destination path should not be a file path {dst}")
-
-        if Path(src).is_file():
-            raise ValueError(f"Source path should not be a file path {src}")
-
-        with ThreadPoolExecutor(max_workers=workers) as executor:
-            futures = []
-            for file in Path(src).rglob("*"):
-                if file.is_file():
-                    dst = UploadPath(
-                        mission=dst.mission,
-                        level=dst.level,
-                        day=dst.day,
-                        month=dst.month,
-                        year=dst.year,
-                        id=dst.id,
-                        path=str(file.relative_to(src)),
-                    )
-                    futures.append(executor.submit(self.upload_file, str(file), dst))
-            for future in futures:
-                future.result()
-
-    @staticmethod
-    def _load_item(item_json_file_path: str) -> DatacosmosItem:
-        """Loads and validates the STAC item from a JSON file."""
-        with open(item_json_file_path, "rb") as file:
-            data = file.read().decode("utf-8")
-        return TypeAdapter(DatacosmosItem).validate_json(data)
-
-    def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
-        """Deletes an existing item if it already exists."""
-        try:
-            self.item_client.delete_item(item_id, collection_id)
-        except Exception:  # nosec
-            pass  # Ignore if item doesn't exist
-
-    def _get_upload_path(self, item: DatacosmosItem, mission_name: str = "") -> str:
-        """Constructs the storage upload path based on the item and mission name."""
-        return UploadPath.from_item_path(item, mission_name, "")
-
-    def _update_item_assets(self, item: DatacosmosItem) -> None:
-        """Updates the item's assets with uploaded file URLs."""
-        for asset in item.assets.values():
-            try:
-                url = self.base_url
-                asset.href = url.with_base(asset.href)  # type: ignore
-            except ValueError:
-                pass