datacosmos 0.0.19__tar.gz → 0.0.21__tar.gz

{datacosmos-0.0.19 → datacosmos-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.19
+Version: 0.0.21
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.19 → datacosmos-0.0.21}/datacosmos/config/auth/factory.py

@@ -5,11 +5,6 @@ This module normalizes the `authentication` config into a concrete model:
 - `apply_auth_defaults` fills sensible defaults per auth type without inventing secrets.
 - `check_required_auth_fields` enforces the minimum required inputs.
 - `normalize_authentication` runs the whole pipeline.
-
-Design notes:
-- Auth models accept partial data (fields are Optional with None defaults).
-- We DO NOT pass `None` explicitly when constructing models here.
-- Required-ness is enforced centrally by `check_required_auth_fields`, not by model init.
 """
 
 from typing import Optional, Union, cast
@@ -33,49 +28,45 @@ AuthModel = Union[M2MAuthenticationConfig, LocalUserAccountAuthenticationConfig]
 
 
 def parse_auth_config(raw: dict | AuthModel | None) -> Optional[AuthModel]:
-    """Turn a raw dict (e.g., from YAML) into a concrete auth model.
-
-    - If `raw` is already an auth model (M2M or local), return it unchanged.
-    - If `raw` is a dict, choose/validate the type using `raw['type']`
-      (or DEFAULT_AUTH_TYPE), then construct the corresponding model.
-      For missing fields we *may* apply non-secret defaults (endpoints, etc.).
-    """
-    if raw is None or isinstance(
-        raw, (M2MAuthenticationConfig, LocalUserAccountAuthenticationConfig)
-    ):
+    """Turn a raw dict (e.g., from YAML/env) into a concrete auth model."""
+    if isinstance(raw, (M2MAuthenticationConfig, LocalUserAccountAuthenticationConfig)):
         return cast(Optional[AuthModel], raw)
 
-    auth_type = _normalize_auth_type(raw.get("type") or DEFAULT_AUTH_TYPE)
+    if raw is None:
+        raw_data = {}
+    else:
+        raw_data = raw.copy()
+
+    if raw is None and not raw_data:
+        return None
+
+    auth_type = _normalize_auth_type(raw_data.get("type") or DEFAULT_AUTH_TYPE)
 
     if auth_type == "local":
         return LocalUserAccountAuthenticationConfig(
             type="local",
-            client_id=raw.get("client_id"),
-            authorization_endpoint=raw.get(
+            client_id=raw_data.get("client_id"),
+            authorization_endpoint=raw_data.get(
                 "authorization_endpoint", DEFAULT_LOCAL_AUTHORIZATION_ENDPOINT
             ),
-            token_endpoint=raw.get("token_endpoint", DEFAULT_LOCAL_TOKEN_ENDPOINT),
-            redirect_port=raw.get("redirect_port", DEFAULT_LOCAL_REDIRECT_PORT),
-            scopes=raw.get("scopes", DEFAULT_LOCAL_SCOPES),
-            audience=raw.get("audience", DEFAULT_AUTH_AUDIENCE),
-            cache_file=raw.get("cache_file", DEFAULT_LOCAL_CACHE_FILE),
+            token_endpoint=raw_data.get("token_endpoint", DEFAULT_LOCAL_TOKEN_ENDPOINT),
+            redirect_port=raw_data.get("redirect_port", DEFAULT_LOCAL_REDIRECT_PORT),
+            scopes=raw_data.get("scopes", DEFAULT_LOCAL_SCOPES),
+            audience=raw_data.get("audience", DEFAULT_AUTH_AUDIENCE),
+            cache_file=raw_data.get("cache_file", DEFAULT_LOCAL_CACHE_FILE),
         )
 
     return M2MAuthenticationConfig(
         type="m2m",
-        token_url=raw.get("token_url", DEFAULT_AUTH_TOKEN_URL),
-        audience=raw.get("audience", DEFAULT_AUTH_AUDIENCE),
-        client_id=raw.get("client_id"),
-        client_secret=raw.get("client_secret"),
+        token_url=raw_data.get("token_url", DEFAULT_AUTH_TOKEN_URL),
+        audience=raw_data.get("audience", DEFAULT_AUTH_AUDIENCE),
+        client_id=raw_data.get("client_id"),
+        client_secret=raw_data.get("client_secret"),
     )
 
 
 def apply_auth_defaults(auth: AuthModel | None) -> AuthModel:
-    """Fill in any missing defaults by type (non-secret values only).
-
-    If `auth` is None, construct a default "shell" based on DEFAULT_AUTH_TYPE,
-    without passing None for unknown credentials.
-    """
+    """Fill in any missing defaults by type (non-secret values only)."""
     if auth is None:
         default_type = _normalize_auth_type(DEFAULT_AUTH_TYPE)
         if default_type == "local":
@@ -101,7 +92,6 @@ def apply_auth_defaults(auth: AuthModel | None) -> AuthModel:
         auth.audience = auth.audience or DEFAULT_AUTH_AUDIENCE
         return auth
 
-    # Local defaults (Pydantic already coerces types; only set when missing)
     auth.type = auth.type or "local"
     auth.authorization_endpoint = (
         auth.authorization_endpoint or DEFAULT_LOCAL_AUTHORIZATION_ENDPOINT
@@ -116,11 +106,7 @@ def apply_auth_defaults(auth: AuthModel | None) -> AuthModel:
 
 
 def check_required_auth_fields(auth: AuthModel) -> None:
-    """Enforce required fields per auth type.
-
-    - m2m requires client_id and client_secret.
-    - local requires client_id.
-    """
+    """Enforce required fields per auth type."""
     if isinstance(auth, M2MAuthenticationConfig):
         missing = [f for f in ("client_id", "client_secret") if not getattr(auth, f)]
         if missing:

{datacosmos-0.0.19 → datacosmos-0.0.21}/datacosmos/config/config.py

@@ -7,7 +7,7 @@ and supports environment variable-based overrides.
 
 from typing import Optional
 
-from pydantic import field_validator
+from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from datacosmos.config.auth.factory import normalize_authentication, parse_auth_config
@@ -18,6 +18,10 @@ from datacosmos.config.constants import (
 )
 from datacosmos.config.loaders.yaml_source import yaml_settings_source
 from datacosmos.config.models.authentication_config import AuthenticationConfig
+from datacosmos.config.models.local_user_account_authentication_config import (
+    LocalUserAccountAuthenticationConfig,
+)
+from datacosmos.config.models.m2m_authentication_config import M2MAuthenticationConfig
 from datacosmos.config.models.url import URL
 
 
@@ -31,9 +35,14 @@ class Config(BaseSettings):
     )
 
     authentication: Optional[AuthenticationConfig] = None
-    stac: URL | None = None
-    datacosmos_cloud_storage: URL | None = None
-    datacosmos_public_cloud_storage: URL | None = None
+
+    stac: URL = Field(default_factory=lambda: URL(**DEFAULT_STAC))
+    datacosmos_cloud_storage: URL = Field(
+        default_factory=lambda: URL(**DEFAULT_STORAGE)
+    )
+    datacosmos_public_cloud_storage: URL = Field(
+        default_factory=lambda: URL(**DEFAULT_STORAGE)
+    )
 
     @classmethod
     def settings_customise_sources(cls, *args, **kwargs):
@@ -65,13 +74,6 @@ class Config(BaseSettings):
     def _parse_authentication(cls, raw):
         if raw is None:
             return None
-        from datacosmos.config.models.local_user_account_authentication_config import (
-            LocalUserAccountAuthenticationConfig,
-        )
-        from datacosmos.config.models.m2m_authentication_config import (
-            M2MAuthenticationConfig,
-        )
-
         if isinstance(
             raw, (M2MAuthenticationConfig, LocalUserAccountAuthenticationConfig)
         ):
@@ -84,18 +86,3 @@ class Config(BaseSettings):
     @classmethod
     def _validate_authentication(cls, auth: Optional[AuthenticationConfig]):
         return normalize_authentication(auth)
-
-    @field_validator("stac", mode="before")
-    @classmethod
-    def _default_stac(cls, value: URL | None) -> URL:
-        return value or URL(**DEFAULT_STAC)
-
-    @field_validator("datacosmos_cloud_storage", mode="before")
-    @classmethod
-    def _default_cloud_storage(cls, value: URL | None) -> URL:
-        return value or URL(**DEFAULT_STORAGE)
-
-    @field_validator("datacosmos_public_cloud_storage", mode="before")
-    @classmethod
-    def _default_public_cloud_storage(cls, value: URL | None) -> URL:
-        return value or URL(**DEFAULT_STORAGE)

datacosmos-0.0.21/datacosmos/stac/storage/storage_base.py

@@ -0,0 +1,77 @@
+"""Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+import mimetypes
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from typing import Any, Callable, Dict, Iterable, List, Tuple
+
+from datacosmos.datacosmos_client import DatacosmosClient
+from datacosmos.exceptions.datacosmos_error import DatacosmosError
+
+
+class StorageBase:
+    """Base class providing common storage helpers (threading, MIME guess, futures)."""
+
+    def __init__(self, client: DatacosmosClient):
+        """Base class providing common storage helpers (threading, MIME guess, futures)."""
+        self.client = client
+        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
+
+    def _guess_mime(self, src: str) -> str:
+        mime, _ = mimetypes.guess_type(src)
+        return mime or "application/octet-stream"
+
+    def run_in_threads(
+        self,
+        fn: Callable[..., Any],
+        jobs: Iterable[Tuple[Any, ...]],
+        max_workers: int,
+        timeout: float,
+    ) -> Tuple[List[Any], List[Dict[str, Any]]]:
+        """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
+
+        Collects successes and failures without aborting the batch on individual errors.
+
+        Args:
+            fn: The function to execute.
+            jobs: An iterable of tuples, where each tuple is unpacked as fn(*args).
+            max_workers: Maximum number of threads to use.
+            timeout: Timeout for the entire batch.
+
+        Returns:
+            A tuple containing (successes: List[Any], failures: List[Dict[str, Any]]).
+            Failures include the exception and job arguments.
+
+        Raises:
+            DatacosmosError: If the entire batch times out.
+        """
+        futures: List[Future] = []
+
+        executor = ThreadPoolExecutor(max_workers=max_workers)
+
+        try:
+            for args in jobs:
+                futures.append(executor.submit(fn, *args))
+
+            # Wait until all futures are done or the timeout is reached
+            done, not_done = wait(futures, timeout=timeout)
+
+            successes = []
+            failures = []
+
+            for future in done:
+                try:
+                    result = future.result()
+                except Exception as e:
+                    failures.append({"error": str(e), "exception": e})
+                else:
+                    successes.append(result)
+
+            if not_done:
+                # The executor's shutdown wait must be skipped to allow cancellation
+                raise DatacosmosError("Batch processing failed: operation timed out.")
+
+            return successes, failures
+        finally:
+            # Shutdown without waiting to enable timeout handling
+            # The wait call already established which jobs finished
+            executor.shutdown(wait=False)

{datacosmos-0.0.19 → datacosmos-0.0.21}/datacosmos/stac/storage/uploader.py

@@ -1,6 +1,7 @@
 """Handles uploading files to Datacosmos storage and registering STAC items."""
 
 from pathlib import Path
+from typing import Any
 
 from pydantic import TypeAdapter
 
@@ -13,7 +14,7 @@ from datacosmos.stac.storage.storage_base import StorageBase
 
 
 class Uploader(StorageBase):
-    """Upload a STAC item and its assets to Datacosmos storage, then register the item in the STAC API."""
+    """Upload a STAC item and its assets to Datacosmos storage and register the item in the STAC API."""
 
     def __init__(self, client: DatacosmosClient):
         """Initialize the uploader.
@@ -32,17 +33,25 @@ class Uploader(StorageBase):
         included_assets: list[str] | bool = True,
         max_workers: int = 4,
         time_out: float = 60 * 60 * 1,
-    ) -> DatacosmosItem:
-        """Upload a STAC item (and optionally its assets) to Datacosmos.
+    ) -> tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+        """Upload a STAC item (and optionally its assets) to Datacosmos in parallel threads.
 
-        `item` can be either:
-          • a DatacosmosItem instance, or
-          • the path to an item JSON file on disk.
-
-        If `included_assets` is:
-          • True  → upload every asset in the item
-          • list  → upload only the asset keys in that list
-          • False → upload nothing; just register the item
+        Args:
+            item (DatacosmosItem | str):
+                - a DatacosmosItem instance, or
+                - the path to an item JSON file on disk.
+            project_id (str): The project ID to upload assets to.
+            assets_path (str | None): Base directory where local asset files are located.
+            included_assets (list[str] | bool):
+                - True → upload every asset in the item.
+                - list[str] → upload only the asset keys in that list.
+                - False → skip asset upload; just register the item.
+            max_workers (int): Maximum number of parallel threads for asset upload.
+            time_out (float): Timeout in seconds for the entire asset batch upload.
+
+        Returns:
+            tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
+            The updated DatacosmosItem, a list of asset keys that were uploaded successfully, and a list of upload failures.
         """
         if not assets_path and not isinstance(item, str):
             raise ValueError(
@@ -54,23 +63,45 @@ class Uploader(StorageBase):
             item = self._load_item(item_filename)
             assets_path = assets_path or str(Path(item_filename).parent)
 
+        if not isinstance(item, DatacosmosItem):
+            raise TypeError(f"item must be a DatacosmosItem, got {type(item).__name__}")
+
         assets_path = assets_path or str(Path.cwd())
 
-        upload_assets = (
-            included_assets
-            if isinstance(included_assets, list)
-            else item.assets.keys()
-            if included_assets is True
-            else []
-        )
+        if included_assets is False:
+            upload_assets: list[str] = []
+        elif included_assets is True:
+            upload_assets = list(item.assets.keys())
+        elif isinstance(included_assets, list):
+            upload_assets = included_assets
+        else:
+            upload_assets = []
 
         jobs = [
             (item, asset_key, assets_path, project_id) for asset_key in upload_assets
         ]
-        self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
 
-        self.item_client.add_item(item)
-        return item
+        if not jobs:
+            self.item_client.add_item(item)
+            return item, [], []
+
+        successes, failures = self.run_in_threads(
+            self._upload_asset, jobs, max_workers, time_out
+        )
+
+        # Register the item if the overall process didn't time out
+        # and there was at least one successful upload.
+        if successes:
+            self.item_client.add_item(item)
+
+        return item, successes, failures
+
+    @staticmethod
+    def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        """Load a DatacosmosItem from a JSON file on disk."""
+        with open(item_json_file_path, "rb") as file:
+            data = file.read().decode("utf-8")
+        return TypeAdapter(DatacosmosItem).validate_json(data)
 
     def upload_from_file(
         self, src: str, dst: str, mime_type: str | None = None
@@ -83,19 +114,13 @@ class Uploader(StorageBase):
             response = self.client.put(url, data=f, headers=headers)
         response.raise_for_status()
 
-    @staticmethod
-    def _load_item(item_json_file_path: str) -> DatacosmosItem:
-        """Load a DatacosmosItem from a JSON file on disk."""
-        with open(item_json_file_path, "rb") as file:
-            data = file.read().decode("utf-8")
-        return TypeAdapter(DatacosmosItem).validate_json(data)
-
     def _upload_asset(
         self, item: DatacosmosItem, asset_key: str, assets_path: str, project_id: str
-    ) -> None:
+    ) -> str:
         """Upload a single asset file and update its href inside the item object.
 
-        Runs in parallel via _run_in_threads().
+        Returns:
+            str: The asset_key upon successful upload.
         """
         asset = item.assets[asset_key]
 
@@ -117,6 +142,8 @@ class Uploader(StorageBase):
         self._update_asset_href(asset)  # turn href into public URL
         self.upload_from_file(src, str(upload_path), mime_type=asset.type)
 
+        return asset_key
+
     def _update_asset_href(self, asset: Asset) -> None:
         """Convert the storage key to a public HTTPS URL."""
         try:

{datacosmos-0.0.19 → datacosmos-0.0.21}/datacosmos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacosmos
-Version: 0.0.19
+Version: 0.0.21
 Summary: A library for interacting with DataCosmos from Python code
 Author-email: Open Cosmos <support@open-cosmos.com>
 Classifier: Programming Language :: Python :: 3

{datacosmos-0.0.19 → datacosmos-0.0.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datacosmos"
-version = "0.0.19"
+version = "0.0.21"
 authors = [
     { name="Open Cosmos", email="support@open-cosmos.com" },
 ]

datacosmos-0.0.19/datacosmos/stac/storage/storage_base.py

@@ -1,40 +0,0 @@
-"""Base class providing common storage helpers (threading, MIME guess, futures)."""
-
-import mimetypes
-from concurrent.futures import ThreadPoolExecutor, wait
-
-from datacosmos.datacosmos_client import DatacosmosClient
-
-
-class StorageBase:
-    """Base class providing common storage helpers (threading, MIME guess, futures)."""
-
-    def __init__(self, client: DatacosmosClient):
-        """Base class providing common storage helpers (threading, MIME guess, futures)."""
-        self.client = client
-        self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
-
-    def _guess_mime(self, src: str) -> str:
-        mime, _ = mimetypes.guess_type(src)
-        return mime or "application/octet-stream"
-
-    def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
-        """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
-
-        `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
-        """
-        futures = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            for args in fn_args:
-                futures.append(executor.submit(fn, *args))
-        done, not_done = wait(futures, timeout=timeout)
-        errors = []
-        for future in done:
-            try:
-                future.result()
-            except Exception as e:
-                errors.append(e)
-        for future in not_done:
-            future.cancel()
-        if errors:
-            raise errors[0]