datacosmos 0.0.19__py3-none-any.whl → 0.0.20__py3-none-any.whl

datacosmos/stac/storage/storage_base.py

@@ -1,9 +1,11 @@
 """Base class providing common storage helpers (threading, MIME guess, futures)."""
 
 import mimetypes
-from concurrent.futures import ThreadPoolExecutor, wait
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from typing import Any, Callable, Dict, Iterable, List, Tuple
 
 from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.exceptions.datacosmos_error import DatacosmosError
 
 
 class StorageBase:
@@ -18,23 +20,58 @@ class StorageBase:
         mime, _ = mimetypes.guess_type(src)
         return mime or "application/octet-stream"
 
-    def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
+    def run_in_threads(
+        self,
+        fn: Callable[..., Any],
+        jobs: Iterable[Tuple[Any, ...]],
+        max_workers: int,
+        timeout: float,
+    ) -> Tuple[List[Any], List[Dict[str, Any]]]:
         """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
 
-        `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
+        Collects successes and failures without aborting the batch on individual errors.
+
+        Args:
+            fn: The function to execute.
+            jobs: An iterable of tuples, where each tuple is unpacked as fn(*args).
+            max_workers: Maximum number of threads to use.
+            timeout: Timeout for the entire batch.
+
+        Returns:
+            A tuple containing (successes: List[Any], failures: List[Dict[str, Any]]).
+            Failures include the exception and job arguments.
+
+        Raises:
+            DatacosmosError: If the entire batch times out.
         """
-        futures = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            for args in fn_args:
+        futures: List[Future] = []
+
+        executor = ThreadPoolExecutor(max_workers=max_workers)
+
+        try:
+            for args in jobs:
                 futures.append(executor.submit(fn, *args))
-        done, not_done = wait(futures, timeout=timeout)
-        errors = []
-        for future in done:
-            try:
-                future.result()
-            except Exception as e:
-                errors.append(e)
-        for future in not_done:
-            future.cancel()
-        if errors:
-            raise errors[0]
+
+            # Wait until all futures are done or the timeout is reached
+            done, not_done = wait(futures, timeout=timeout)
+
+            successes = []
+            failures = []
+
+            for future in done:
+                try:
+                    result = future.result()
+                except Exception as e:
+                    failures.append({'error': str(e), 'exception': e})
+                else:
+                    successes.append(result)
+
+            if not_done:
+                # The executor's shutdown wait must be skipped to allow cancellation
+                raise DatacosmosError("Batch processing failed: operation timed out.")
+
+            return successes, failures
+        finally:
+            # Shutdown without waiting to enable timeout handling
+            # The wait call already established which jobs finished
+            executor.shutdown(wait=False)

datacosmos/stac/storage/uploader.py

@@ -1,6 +1,7 @@
 """Handles uploading files to Datacosmos storage and registering STAC items."""
 
 from pathlib import Path
+from typing import Any
 
 from pydantic import TypeAdapter
 
@@ -13,7 +14,7 @@ from datacosmos.stac.storage.storage_base import StorageBase
 
 
 class Uploader(StorageBase):
-    """Upload a STAC item and its assets to Datacosmos storage, then register the item in the STAC API."""
+    """Upload a STAC item and its assets to Datacosmos storage and register the item in the STAC API."""
 
     def __init__(self, client: DatacosmosClient):
         """Initialize the uploader.
@@ -32,17 +33,25 @@ class Uploader(StorageBase):
         included_assets: list[str] | bool = True,
         max_workers: int = 4,
         time_out: float = 60 * 60 * 1,
-    ) -> DatacosmosItem:
-        """Upload a STAC item (and optionally its assets) to Datacosmos.
+    ) -> tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+        """Upload a STAC item (and optionally its assets) to Datacosmos in parallel threads.
 
-        `item` can be either:
-          • a DatacosmosItem instance, or
-          • the path to an item JSON file on disk.
-
-        If `included_assets` is:
-          • True  → upload every asset in the item
-          • list  → upload only the asset keys in that list
-          • False → upload nothing; just register the item
+        Args:
+            item (DatacosmosItem | str):
+                - a DatacosmosItem instance, or
+                - the path to an item JSON file on disk.
+            project_id (str): The project ID to upload assets to.
+            assets_path (str | None): Base directory where local asset files are located.
+            included_assets (list[str] | bool):
+                - True → upload every asset in the item.
+                - list[str] → upload only the asset keys in that list.
+                - False → skip asset upload; just register the item.
+            max_workers (int): Maximum number of parallel threads for asset upload.
+            time_out (float): Timeout in seconds for the entire asset batch upload.
+
+        Returns:
+            tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+            The updated DatacosmosItem, a list of asset keys that were uploaded successfully, and a list of upload failures.
         """
         if not assets_path and not isinstance(item, str):
             raise ValueError(
@@ -54,23 +63,45 @@ class Uploader(StorageBase):
             item = self._load_item(item_filename)
             assets_path = assets_path or str(Path(item_filename).parent)
 
+        if not isinstance(item, DatacosmosItem):
+            raise TypeError(f"item must be a DatacosmosItem, got {type(item).__name__}")
+
         assets_path = assets_path or str(Path.cwd())
 
-        upload_assets = (
-            included_assets
-            if isinstance(included_assets, list)
-            else item.assets.keys()
-            if included_assets is True
-            else []
-        )
+        if included_assets is False:
+            upload_assets: list[str] = []
+        elif included_assets is True:
+            upload_assets = list(item.assets.keys())
+        elif isinstance(included_assets, list):
+            upload_assets = included_assets
+        else:
+            upload_assets = []
 
         jobs = [
             (item, asset_key, assets_path, project_id) for asset_key in upload_assets
         ]
-        self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
 
-        self.item_client.add_item(item)
-        return item
+        if not jobs:
+            self.item_client.add_item(item)
+            return item, [], []
+
+        successes, failures = self.run_in_threads(
+            self._upload_asset, jobs, max_workers, time_out
+        )
+
+        # Register the item if the overall process didn't time out
+        # and there was at least one successful upload.
+        if successes:
+            self.item_client.add_item(item)
+
+        return item, successes, failures
+
+    @staticmethod
+    def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        """Load a DatacosmosItem from a JSON file on disk."""
+        with open(item_json_file_path, "rb") as file:
+            data = file.read().decode("utf-8")
+        return TypeAdapter(DatacosmosItem).validate_json(data)
 
     def upload_from_file(
         self, src: str, dst: str, mime_type: str | None = None
@@ -83,19 +114,13 @@ class Uploader(StorageBase):
             response = self.client.put(url, data=f, headers=headers)
         response.raise_for_status()
 
-    @staticmethod
-    def _load_item(item_json_file_path: str) -> DatacosmosItem:
-        """Load a DatacosmosItem from a JSON file on disk."""
-        with open(item_json_file_path, "rb") as file:
-            data = file.read().decode("utf-8")
-        return TypeAdapter(DatacosmosItem).validate_json(data)
-
     def _upload_asset(
         self, item: DatacosmosItem, asset_key: str, assets_path: str, project_id: str
-    ) -> None:
+    ) -> str:
         """Upload a single asset file and update its href inside the item object.
 
-        Runs in parallel via _run_in_threads().
+        Returns:
+            str: The asset_key upon successful upload.
         """
         asset = item.assets[asset_key]
 
@@ -117,6 +142,8 @@ class Uploader(StorageBase):
         self._update_asset_href(asset)  # turn href into public URL
         self.upload_from_file(src, str(upload_path), mime_type=asset.type)
 
+        return asset_key
+
     def _update_asset_href(self, asset: Asset) -> None:
         """Convert the storage key to a public HTTPS URL."""
         try:

{datacosmos-0.0.19.dist-info → datacosmos-0.0.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.19
+Version: 0.0.20
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.19.dist-info → datacosmos-0.0.20.dist-info}/RECORD

@@ -43,9 +43,9 @@ datacosmos/stac/item/models/eo_band.py,sha256=YC3Scn_wFhIo51pIVcJeuJienF7JGWoEv3
 datacosmos/stac/item/models/item_update.py,sha256=_CpjQn9SsfedfuxlHSiGeptqY4M-p15t9YX__mBRueI,2088
 datacosmos/stac/item/models/raster_band.py,sha256=CoEVs-YyPE5Fse0He9DdOs4dGZpzfCsCuVzOcdXa_UM,354
 datacosmos/stac/storage/__init__.py,sha256=hivfSpOaoSwCAymgU0rTgvSk9LSPAn1cPLQQ9fLmFX0,151
-datacosmos/stac/storage/storage_base.py,sha256=5ioMKbEltPEWr4dkhZQiUhdBFEhe7ajIYUd9z3K8elU,1483
+datacosmos/stac/storage/storage_base.py,sha256=NpCKAA3qEI212WUNZ2-eG9XfSJKDMiJcu4pEmr10-JI,2843
 datacosmos/stac/storage/storage_client.py,sha256=4boqQ3zVMrk9X2IXus-Cs429juLe0cUQ0XEzg_y3yOA,1205
-datacosmos/stac/storage/uploader.py,sha256=cUe-6DbcEXcENReDMaQe4etVCW1n2kPMxQlCuB8YnqU,4635
+datacosmos/stac/storage/uploader.py,sha256=DawtNn4-uEtpUYPZS1fKv77wu-zNne9ltlGZiArzBFI,5919
 datacosmos/stac/storage/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
 datacosmos/stac/storage/dataclasses/upload_path.py,sha256=gbpV67FECFNyXn-yGUSuLvGGWHtibbZq7Qu9yGod3C0,1398
 datacosmos/utils/__init__.py,sha256=XQbAnoqJrPpnSpEzAbjh84yqYWw8cBM8mNp8ynTG-54,50
@@ -55,8 +55,8 @@ datacosmos/utils/http_response/check_api_response.py,sha256=l_yQiiekNcNbhFec_5Ue
 datacosmos/utils/http_response/models/__init__.py,sha256=Wj8YT6dqw7rAz_rctllxo5Or_vv8DwopvQvBzwCTvpw,45
 datacosmos/utils/http_response/models/datacosmos_error.py,sha256=Uqi2uM98nJPeCbM7zngV6vHSk97jEAb_nkdDEeUjiQM,740
 datacosmos/utils/http_response/models/datacosmos_response.py,sha256=oV4n-sue7K1wwiIQeHpxdNU8vxeqF3okVPE2rydw5W0,336
-datacosmos-0.0.19.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
-datacosmos-0.0.19.dist-info/METADATA,sha256=K01wNYqlqufpA_-cpPIF4M_ccP9tVmmYMGE5PLf2LOk,1000
-datacosmos-0.0.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datacosmos-0.0.19.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
-datacosmos-0.0.19.dist-info/RECORD,,
+datacosmos-0.0.20.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
+datacosmos-0.0.20.dist-info/METADATA,sha256=WfpmuajQ7GSenjnr8X04rNXDg6U6NH3pmbFMtJYqFV4,1000
+datacosmos-0.0.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datacosmos-0.0.20.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
+datacosmos-0.0.20.dist-info/RECORD,,