datacosmos 0.0.19__tar.gz → 0.0.20__tar.gz

{datacosmos-0.0.19 → datacosmos-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.19
+Version: 0.0.20
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

datacosmos-0.0.20/datacosmos/stac/storage/storage_base.py

@@ -0,0 +1,77 @@
+"""Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+import mimetypes
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from typing import Any, Callable, Dict, Iterable, List, Tuple
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.exceptions.datacosmos_error import DatacosmosError
+
+
+class StorageBase:
+    """Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Base class providing common storage helpers (threading, MIME guess, futures)."""
+        self.client = client
+        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
+
+    def _guess_mime(self, src: str) -> str:
+        mime, _ = mimetypes.guess_type(src)
+        return mime or "application/octet-stream"
+
+    def run_in_threads(
+        self,
+        fn: Callable[..., Any],
+        jobs: Iterable[Tuple[Any, ...]],
+        max_workers: int,
+        timeout: float,
+    ) -> Tuple[List[Any], List[Dict[str, Any]]]:
+        """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
+
+        Collects successes and failures without aborting the batch on individual errors.
+
+        Args:
+            fn: The function to execute.
+            jobs: An iterable of tuples, where each tuple is unpacked as fn(*args).
+            max_workers: Maximum number of threads to use.
+            timeout: Timeout for the entire batch.
+
+        Returns:
+            A tuple containing (successes: List[Any], failures: List[Dict[str, Any]]).
+            Failures include the exception and job arguments.
+
+        Raises:
+            DatacosmosError: If the entire batch times out.
+        """
+        futures: List[Future] = []
+
+        executor = ThreadPoolExecutor(max_workers=max_workers)
+
+        try:
+            for args in jobs:
+                futures.append(executor.submit(fn, *args))
+
+            # Wait until all futures are done or the timeout is reached
+            done, not_done = wait(futures, timeout=timeout)
+
+            successes = []
+            failures = []
+
+            for future in done:
+                try:
+                    result = future.result()
+                except Exception as e:
+                    failures.append({'error': str(e), 'exception': e})
+                else:
+                    successes.append(result)
+
+            if not_done:
+                # The executor's shutdown wait must be skipped to allow cancellation
+                raise DatacosmosError("Batch processing failed: operation timed out.")
+
+            return successes, failures
+        finally:
+            # Shutdown without waiting to enable timeout handling
+            # The wait call already established which jobs finished
+            executor.shutdown(wait=False)

{datacosmos-0.0.19 → datacosmos-0.0.20}/datacosmos/stac/storage/uploader.py

@@ -1,6 +1,7 @@
 """Handles uploading files to Datacosmos storage and registering STAC items."""
 
 from pathlib import Path
+from typing import Any
 
 from pydantic import TypeAdapter
 
@@ -13,7 +14,7 @@ from datacosmos.stac.storage.storage_base import StorageBase
 
 
 class Uploader(StorageBase):
-    """Upload a STAC item and its assets to Datacosmos storage, then register the item in the STAC API."""
+    """Upload a STAC item and its assets to Datacosmos storage and register the item in the STAC API."""
 
     def __init__(self, client: DatacosmosClient):
         """Initialize the uploader.
@@ -32,17 +33,25 @@ class Uploader(StorageBase):
         included_assets: list[str] | bool = True,
         max_workers: int = 4,
         time_out: float = 60 * 60 * 1,
-    ) -> DatacosmosItem:
-        """Upload a STAC item (and optionally its assets) to Datacosmos.
+    ) -> tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+        """Upload a STAC item (and optionally its assets) to Datacosmos in parallel threads.
 
-        `item` can be either:
-          • a DatacosmosItem instance, or
-          • the path to an item JSON file on disk.
-
-        If `included_assets` is:
-          • True  → upload every asset in the item
-          • list  → upload only the asset keys in that list
-          • False → upload nothing; just register the item
+        Args:
+            item (DatacosmosItem | str):
+                - a DatacosmosItem instance, or
+                - the path to an item JSON file on disk.
+            project_id (str): The project ID to upload assets to.
+            assets_path (str | None): Base directory where local asset files are located.
+            included_assets (list[str] | bool):
+                - True → upload every asset in the item.
+                - list[str] → upload only the asset keys in that list.
+                - False → skip asset upload; just register the item.
+            max_workers (int): Maximum number of parallel threads for asset upload.
+            time_out (float): Timeout in seconds for the entire asset batch upload.
+
+        Returns:
+            tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+            The updated DatacosmosItem, a list of asset keys that were uploaded successfully, and a list of upload failures.
         """
         if not assets_path and not isinstance(item, str):
             raise ValueError(
@@ -54,23 +63,45 @@ class Uploader(StorageBase):
             item = self._load_item(item_filename)
             assets_path = assets_path or str(Path(item_filename).parent)
 
+        if not isinstance(item, DatacosmosItem):
+            raise TypeError(f"item must be a DatacosmosItem, got {type(item).__name__}")
+
         assets_path = assets_path or str(Path.cwd())
 
-        upload_assets = (
-            included_assets
-            if isinstance(included_assets, list)
-            else item.assets.keys()
-            if included_assets is True
-            else []
-        )
+        if included_assets is False:
+            upload_assets: list[str] = []
+        elif included_assets is True:
+            upload_assets = list(item.assets.keys())
+        elif isinstance(included_assets, list):
+            upload_assets = included_assets
+        else:
+            upload_assets = []
 
         jobs = [
             (item, asset_key, assets_path, project_id) for asset_key in upload_assets
         ]
-        self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
 
-        self.item_client.add_item(item)
-        return item
+        if not jobs:
+            self.item_client.add_item(item)
+            return item, [], []
+
+        successes, failures = self.run_in_threads(
+            self._upload_asset, jobs, max_workers, time_out
+        )
+
+        # Register the item if the overall process didn't time out
+        # and there was at least one successful upload.
+        if successes:
+            self.item_client.add_item(item)
+
+        return item, successes, failures
+
+    @staticmethod
+    def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        """Load a DatacosmosItem from a JSON file on disk."""
+        with open(item_json_file_path, "rb") as file:
+            data = file.read().decode("utf-8")
+        return TypeAdapter(DatacosmosItem).validate_json(data)
 
     def upload_from_file(
         self, src: str, dst: str, mime_type: str | None = None
@@ -83,19 +114,13 @@ class Uploader(StorageBase):
             response = self.client.put(url, data=f, headers=headers)
         response.raise_for_status()
 
-    @staticmethod
-    def _load_item(item_json_file_path: str) -> DatacosmosItem:
-        """Load a DatacosmosItem from a JSON file on disk."""
-        with open(item_json_file_path, "rb") as file:
-            data = file.read().decode("utf-8")
-        return TypeAdapter(DatacosmosItem).validate_json(data)
-
     def _upload_asset(
         self, item: DatacosmosItem, asset_key: str, assets_path: str, project_id: str
-    ) -> None:
+    ) -> str:
         """Upload a single asset file and update its href inside the item object.
 
-        Runs in parallel via _run_in_threads().
+        Returns:
+            str: The asset_key upon successful upload.
         """
         asset = item.assets[asset_key]
 
@@ -117,6 +142,8 @@ class Uploader(StorageBase):
         self._update_asset_href(asset)  # turn href into public URL
         self.upload_from_file(src, str(upload_path), mime_type=asset.type)
 
+        return asset_key
+
     def _update_asset_href(self, asset: Asset) -> None:
         """Convert the storage key to a public HTTPS URL."""
         try:

{datacosmos-0.0.19 → datacosmos-0.0.20}/datacosmos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.19
+Version: 0.0.20
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.19 → datacosmos-0.0.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datacosmos"
-version = "0.0.19"
+version = "0.0.20"
 authors = [
     { name="Open Cosmos", email="support@open-cosmos.com" },
 ]

datacosmos-0.0.19/datacosmos/stac/storage/storage_base.py

@@ -1,40 +0,0 @@
-"""Base class providing common storage helpers (threading, MIME guess, futures)."""
-
-import mimetypes
-from concurrent.futures import ThreadPoolExecutor, wait
-
-from datacosmos.datacosmos_client import DatacosmosClient
-
-
-class StorageBase:
-    """Base class providing common storage helpers (threading, MIME guess, futures)."""
-
-    def __init__(self, client: DatacosmosClient):
-        """Base class providing common storage helpers (threading, MIME guess, futures)."""
-        self.client = client
-        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
-
-    def _guess_mime(self, src: str) -> str:
-        mime, _ = mimetypes.guess_type(src)
-        return mime or "application/octet-stream"
-
-    def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
-        """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
-
-        `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
-        """
-        futures = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            for args in fn_args:
-                futures.append(executor.submit(fn, *args))
-        done, not_done = wait(futures, timeout=timeout)
-        errors = []
-        for future in done:
-            try:
-                future.result()
-            except Exception as e:
-                errors.append(e)
-        for future in not_done:
-            future.cancel()
-        if errors:
-            raise errors[0]