datacosmos 0.0.11__py3-none-any.whl → 0.0.13__py3-none-any.whl

datacosmos/config/constants.py

@@ -0,0 +1,26 @@
+"""Config constants."""
+
+# ---- Authentication defaults ----
+DEFAULT_AUTH_TYPE = "m2m"
+
+# M2M
+DEFAULT_AUTH_TOKEN_URL = "https://login.open-cosmos.com/oauth/token"
+DEFAULT_AUTH_AUDIENCE = "https://beeapp.open-cosmos.com"
+
+# Local (interactive)
+DEFAULT_LOCAL_AUTHORIZATION_ENDPOINT = "https://login.open-cosmos.com/authorize"
+DEFAULT_LOCAL_TOKEN_ENDPOINT = DEFAULT_AUTH_TOKEN_URL
+DEFAULT_LOCAL_REDIRECT_PORT = 8765
+DEFAULT_LOCAL_SCOPES = "openid profile email offline_access"
+DEFAULT_LOCAL_CACHE_FILE = "~/.datacosmos/token_cache.json"
+
+# ---- Service URLs ----
+DEFAULT_STAC = dict(
+    protocol="https", host="app.open-cosmos.com", port=443, path="/api/data/v0/stac"
+)
+DEFAULT_STORAGE = dict(
+    protocol="https", host="app.open-cosmos.com", port=443, path="/api/data/v0/storage"
+)
+
+# ---- Config file path ----
+DEFAULT_CONFIG_YAML = "config/config.yaml"

datacosmos/config/loaders/yaml_source.py

@@ -0,0 +1,62 @@
+"""YAML settings source for pydantic-settings.
+
+This module provides a tiny helper to inject a YAML file as a configuration
+source for `pydantic-settings` (v2.x). It returns a callable compatible with
+`BaseSettings.settings_customise_sources`, placed wherever you want in the
+precedence chain.
+
+- If the file is missing, the source returns an empty dict.
+- Empty-ish values (`None`, empty string, empty list) are dropped so they don't
+  overwrite values coming from later sources (e.g., environment variables).
+- The returned callable accepts `*args, **kwargs` to be version-agnostic across
+  pydantic-settings minor releases (some pass positional args, others keywords).
+"""
+
+from typing import Any, Callable, Dict
+
+# A callable that returns a mapping of settings values when invoked by pydantic-settings.
+SettingsSourceCallable = Callable[..., Dict[str, Any]]
+
+
+def yaml_settings_source(file_path: str) -> SettingsSourceCallable:
+    """Create a pydantic-settings-compatible source that reads from a YAML file.
+
+    Parameters
+    ----------
+    file_path : str
+        Absolute or relative path to the YAML file to load.
+
+    Returns
+    -------
+    SettingsSourceCallable
+        A callable that, when invoked by pydantic-settings, returns a dict
+        of settings loaded from the YAML file. If the file does not exist,
+        an empty dict is returned. Keys with empty/None values are omitted
+        so later sources (e.g., env vars) can provide effective overrides.
+
+    Notes
+    -----
+    The returned callable accepts arbitrary `*args` and `**kwargs` to stay
+    compatible with different pydantic-settings 2.x calling conventions.
+    """
+
+    def _source(*_args: Any, **_kwargs: Any) -> Dict[str, Any]:
+        """Load and sanitize YAML content for use as a settings source.
+
+        Returns
+        -------
+        dict
+            A dictionary of settings. If the YAML file is missing, `{}`.
+            Values that are `None`, empty strings, or empty lists are dropped.
+        """
+        import os
+
+        import yaml
+
+        if not os.path.exists(file_path):
+            return {}
+        with open(file_path, "r") as f:
+            data = yaml.safe_load(f) or {}
+        return {k: v for k, v in data.items() if v not in (None, "", [])}
+
+    return _source

datacosmos/config/models/local_user_account_authentication_config.py

@@ -4,23 +4,25 @@ When this is chosen, the user will be prompted to log in using their OPS credent
 This will be used for running scripts locally.
 """
 
-from typing import Literal
+from typing import Literal, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class LocalUserAccountAuthenticationConfig(BaseModel):
     """Configuration for local user account authentication.
 
     When this is chosen, the user will be prompted to log in using their OPS credentials.
-    This will be used for running scripts locally.
+    This will be used for running scripts locally. Required fields are enforced by `normalize_authentication` after merge.
     """
 
-    type: Literal["local"]
-    client_id: str
-    authorization_endpoint: str
-    token_endpoint: str
-    redirect_port: int
-    scopes: str
-    audience: str
-    cache_file: str
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["local"] = "local"
+    client_id: Optional[str] = None
+    authorization_endpoint: Optional[str] = None
+    token_endpoint: Optional[str] = None
+    redirect_port: Optional[int] = None
+    scopes: Optional[str] = None
+    audience: Optional[str] = None
+    cache_file: Optional[str] = None

datacosmos/config/models/m2m_authentication_config.py

@@ -4,24 +4,22 @@ Used when running scripts in the cluster that require automated authentication
 without user interaction.
 """
 
-from typing import Literal
+from typing import Literal, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict
 
 
 class M2MAuthenticationConfig(BaseModel):
     """Configuration for machine-to-machine authentication.
 
     This is used when running scripts in the cluster that require authentication
-    with client credentials.
+    with client credentials. Required fields are enforced by `normalize_authentication` after merge.
     """
 
-    DEFAULT_TYPE: Literal["m2m"] = "m2m"
-    DEFAULT_TOKEN_URL: str = "https://login.open-cosmos.com/oauth/token"
-    DEFAULT_AUDIENCE: str = "https://beeapp.open-cosmos.com"
+    model_config = ConfigDict(extra="forbid")
 
-    type: Literal["m2m"] = Field(default=DEFAULT_TYPE)
-    client_id: str
-    token_url: str = Field(default=DEFAULT_TOKEN_URL)
-    audience: str = Field(default=DEFAULT_AUDIENCE)
-    client_secret: str
+    type: Literal["m2m"] = "m2m"
+    client_id: Optional[str] = None
+    client_secret: Optional[str] = None
+    token_url: Optional[str] = None
+    audience: Optional[str] = None

datacosmos/datacosmos_client.py

@@ -1,6 +1,7 @@
 """Client to interact with the Datacosmos API with authentication and request handling."""
 
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from typing import Any, Optional
 
 import requests
@@ -23,86 +24,183 @@ class DatacosmosClient:
         """Initialize the DatacosmosClient.
 
         Args:
-            config (Optional[Config]): Configuration object (only needed when SDK creates its own session).
-            http_session (Optional[requests.Session]): Pre-authenticated session.
+            config: SDK configuration (if omitted, Config() loads YAML + env).
+            http_session: Pre-authenticated session (OAuth2Session or requests.Session
+                          with 'Authorization: Bearer ...').
         """
+        self.config = self._coerce_config(config)
+        self.token: Optional[str] = None
+        self.token_expiry: Optional[datetime] = None
+
         if http_session is not None:
-            self._http_client = http_session
-            self._owns_session = False
-            if isinstance(http_session, OAuth2Session):
-                token_data = http_session.token
-            elif isinstance(http_session, requests.Session):
-                auth_header = http_session.headers.get("Authorization", "")
-                if not auth_header.startswith("Bearer "):
-                    raise DatacosmosException(
-                        "Injected requests.Session must include a 'Bearer' token in its headers"
-                    )
-                token_data = {"access_token": auth_header.split(" ", 1)[1]}
-            else:
+            self._init_with_injected_session(http_session)
+            return
+
+        self._owns_session = True
+        self._http_client = self._authenticate_and_initialize_client()
+
+    # --------------------------- init helpers ---------------------------
+
+    def _coerce_config(self, cfg: Optional[Config | Any]) -> Config:
+        """Normalize various config inputs into a Config instance."""
+        if cfg is None:
+            return Config()
+        if isinstance(cfg, Config):
+            return cfg
+        if isinstance(cfg, dict):
+            return Config(**cfg)
+        try:
+            return Config.model_validate(cfg)  # pydantic v2
+        except Exception as e:
+            raise DatacosmosException(
+                "Invalid config provided to DatacosmosClient"
+            ) from e
+
+    def _init_with_injected_session(
+        self, http_session: requests.Session | OAuth2Session
+    ) -> None:
+        """Adopt a caller-provided session and extract token/expiry."""
+        self._http_client = http_session
+        self._owns_session = False
+
+        token_data = self._extract_token_data(http_session)
+        self.token = token_data.get("access_token")
+        if not self.token:
+            raise DatacosmosException(
+                "Failed to extract access token from injected session"
+            )
+
+        self.token_expiry = self._compute_expiry(
+            token_data.get("expires_at"),
+            token_data.get("expires_in"),
+        )
+
+    def _extract_token_data(
+        self, http_session: requests.Session | OAuth2Session
+    ) -> dict:
+        """Return {'access_token', 'expires_at'?, 'expires_in'?} from the session."""
+        if isinstance(http_session, OAuth2Session):
+            return getattr(http_session, "token", {}) or {}
+
+        if isinstance(http_session, requests.Session):
+            auth_header = http_session.headers.get("Authorization", "")
+            if not auth_header.startswith("Bearer "):
                 raise DatacosmosException(
-                    f"Unsupported session type: {type(http_session)}"
+                    "Injected requests.Session must include a 'Bearer' token in its headers"
                 )
+            return {"access_token": auth_header.split(" ", 1)[1]}
+
+        raise DatacosmosException(f"Unsupported session type: {type(http_session)}")
+
+    def _compute_expiry(
+        self,
+        expires_at: Optional[datetime | int | float],
+        expires_in: Optional[int | float],
+    ) -> Optional[datetime]:
+        """Normalize expiry inputs to an absolute UTC datetime (or None)."""
+        if isinstance(expires_at, datetime):
+            return expires_at
+        if isinstance(expires_at, (int, float)):
+            return datetime.fromtimestamp(expires_at, tz=timezone.utc)
+        if expires_in is not None:
             try:
-                self.token = token_data.get("access_token")
-                self.token_expiry = token_data.get("expires_at") or token_data.get(
-                    "expires_in"
-                )
-            except Exception:
-                raise DatacosmosException(
-                    "Failed to extract token from injected session"
-                )
+                return datetime.now(timezone.utc) + timedelta(seconds=int(expires_in))
+            except (TypeError, ValueError):
+                return None
+        return None
 
-            self.config = config
-        else:
-            if config:
-                self.config = config
-            else:
-                try:
-                    self.config = Config.from_yaml()
-                except ValueError:
-                    self.config = Config.from_env()
-
-            self._owns_session = True
-            self.token = None
-            self.token_expiry = None
-            self._http_client = self._authenticate_and_initialize_client()
+    # --------------------------- auth/session ---------------------------
 
     def _authenticate_and_initialize_client(self) -> requests.Session:
         """Authenticate and initialize the HTTP client with a valid token."""
+        auth = self.config.authentication
+        auth_type = getattr(auth, "type", "m2m")
+
+        if auth_type == "m2m":
+            return self.__build_m2m_session()
+
+        if auth_type == "local":
+            return self.__build_local_session()
+
+        raise DatacosmosException(f"Unsupported authentication type: {auth_type}")
+
+    def _refresh_token_if_needed(self):
+        """Refresh the token if it has expired (only if SDK created it)."""
+        if not getattr(self, "_owns_session", False):
+            return
+        now = datetime.now(timezone.utc)
+        # Treat missing token or missing expiry as 'needs refresh'
+        if (
+            (not self.token)
+            or (self.token_expiry is None)
+            or (self.token_expiry <= now)
+        ):
+            self._http_client = self._authenticate_and_initialize_client()
+
+    def __build_m2m_session(self) -> requests.Session:
+        """Client Credentials (M2M) flow using requests-oauthlib."""
+        auth = self.config.authentication
         try:
-            client = BackendApplicationClient(
-                client_id=self.config.authentication.client_id
-            )
+            client = BackendApplicationClient(client_id=auth.client_id)
             oauth_session = OAuth2Session(client=client)
 
             token_response = oauth_session.fetch_token(
-                token_url=self.config.authentication.token_url,
-                client_id=self.config.authentication.client_id,
-                client_secret=self.config.authentication.client_secret,
-                audience=self.config.authentication.audience,
+                token_url=auth.token_url,
+                client_id=auth.client_id,
+                client_secret=auth.client_secret,
+                audience=auth.audience,
             )
 
             self.token = token_response["access_token"]
-            self.token_expiry = datetime.now(timezone.utc) + timedelta(
-                seconds=token_response.get("expires_in", 3600)
-            )
+            expires_at = token_response.get("expires_at")
+            if isinstance(expires_at, (int, float)):
+                self.token_expiry = datetime.fromtimestamp(expires_at, tz=timezone.utc)
+            else:
+                self.token_expiry = datetime.now(timezone.utc) + timedelta(
+                    seconds=int(token_response.get("expires_in", 3600))
+                )
 
             http_client = requests.Session()
             http_client.headers.update({"Authorization": f"Bearer {self.token}"})
             return http_client
+
         except (HTTPError, ConnectionError, Timeout) as e:
-            raise DatacosmosException(f"Authentication failed: {str(e)}") from e
+            raise DatacosmosException(f"Authentication failed: {e}") from e
         except RequestException as e:
             raise DatacosmosException(
-                f"Unexpected request failure during authentication: {str(e)}"
+                f"Unexpected request failure during authentication: {e}"
             ) from e
 
-    def _refresh_token_if_needed(self):
-        """Refresh the token if it has expired (only if SDK created it)."""
-        if self._owns_session and (
-            not self.token or self.token_expiry <= datetime.now(timezone.utc)
-        ):
-            self._http_client = self._authenticate_and_initialize_client()
+    def __build_local_session(self) -> requests.Session:
+        """Interactive local login via LocalTokenFetcher (cached + refresh)."""
+        auth = self.config.authentication
+        try:
+            from datacosmos.auth.local_token_fetcher import LocalTokenFetcher
+
+            fetcher = LocalTokenFetcher(
+                client_id=auth.client_id,
+                authorization_endpoint=auth.authorization_endpoint,
+                token_endpoint=auth.token_endpoint,
+                redirect_port=int(auth.redirect_port),
+                audience=auth.audience,
+                scopes=auth.scopes,
+                token_file=Path(auth.cache_file).expanduser(),
+            )
+            tok = fetcher.get_token()
+        except Exception as e:
+            raise DatacosmosException(f"Local authentication failed: {e}") from e
+
+        self.token = tok.access_token
+        self.token_expiry = datetime.fromtimestamp(tok.expires_at, tz=timezone.utc)
+
+        http_client = requests.Session()
+        http_client.headers.update({"Authorization": f"Bearer {self.token}"})
+
+        # keep for potential reuse in refresh path (optional)
+        self._local_token_fetcher = fetcher
+        return http_client
+
+    # --------------------------- request API ---------------------------
 
     def request(
         self, method: str, url: str, *args: Any, **kwargs: Any

datacosmos/stac/stac_client.py

@@ -1,5 +1,6 @@
 """Unified interface for STAC API, combining Item & Collection operations."""
 
+from datacosmos.datacosmos_client import DatacosmosClient
 from datacosmos.stac.collection.collection_client import CollectionClient
 from datacosmos.stac.item.item_client import ItemClient
 from datacosmos.stac.storage.storage_client import StorageClient
@@ -8,7 +9,7 @@ from datacosmos.stac.storage.storage_client import StorageClient
 class STACClient(ItemClient, CollectionClient, StorageClient):
     """Unified interface for STAC API, combining Item & Collection operations."""
 
-    def __init__(self, client):
+    def __init__(self, client: DatacosmosClient):
         """Initialize the STACClient with a DatacosmosClient."""
         ItemClient.__init__(self, client)
         CollectionClient.__init__(self, client)

datacosmos/stac/storage/dataclasses/upload_path.py

@@ -1,63 +1,42 @@
-"""Dataclass for retrieving the upload path of a file."""
+"""Dataclass for generating the upload key of an asset."""
 
 from dataclasses import dataclass
-from datetime import datetime
 from pathlib import Path
 
-import structlog
-
-from datacosmos.stac.enums.processing_level import ProcessingLevel
 from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
 
-logger = structlog.get_logger()
-
 
 @dataclass
 class UploadPath:
-    """Dataclass for retrieving the upload path of a file."""
+    """Storage key in the form: project/<project-id>/<item-id>/<asset-name>."""
 
-    mission: str
-    level: ProcessingLevel
-    day: int
-    month: int
-    year: int
-    id: str
-    path: str
+    project_id: str
+    item_id: str
+    asset_name: str
 
-    def __str__(self):
-        """Return a human-readable string representation of the Path."""
-        path = f"full/{self.mission.lower()}/{self.level.value.lower()}/{self.year:02}/{self.month:02}/{self.day:02}/{self.id}/{self.path}"
-        return path.removesuffix("/")
+    def __str__(self) -> str:
+        """Path in the form: project/<project-id>/<item-id>/<asset-name>."""
+        return f"project/{self.project_id}/{self.item_id}/{self.asset_name}".rstrip("/")
 
     @classmethod
     def from_item_path(
-        cls, item: DatacosmosItem, mission: str, item_path: str
-    ) -> "Path":
-        """Create a Path instance from a DatacosmosItem and a path."""
-        dt = datetime.strptime(item.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
-        path = UploadPath(
-            mission=mission,
-            level=ProcessingLevel(item.properties["processing:level"]),
-            day=dt.day,
-            month=dt.month,
-            year=dt.year,
-            id=item.id,
-            path=item_path,
-        )
-        return cls(**path.__dict__)
+        cls,
+        item: DatacosmosItem,
+        project_id: str,
+        asset_name: str,
+    ) -> "UploadPath":
+        """Create an UploadPath for the given item/asset."""
+        return cls(project_id=project_id, item_id=item.id, asset_name=asset_name)
 
     @classmethod
-    def from_path(cls, path: str) -> "Path":
-        """Create a Path instance from a string path."""
-        parts = path.split("/")
-        if len(parts) < 7:
-            raise ValueError(f"Invalid path {path}")
-        return cls(
-            mission=parts[0],
-            level=ProcessingLevel(parts[1]),
-            day=int(parts[4]),
-            month=int(parts[3]),
-            year=int(parts[2]),
-            id=parts[5],
-            path="/".join(parts[6:]),
-        )
+    def from_path(cls, path: str) -> "UploadPath":
+        """Reverse-parse a storage key back into its components."""
+        parts = Path(path).parts
+        if len(parts) < 4 or parts[0] != "project":
+            raise ValueError(f"Invalid path: {path}")
+
+        project_id, item_id, *rest = parts[1:]
+        asset_name = "/".join(rest)
+        if not asset_name:
+            raise ValueError(f"Asset name is missing in path: {path}")
+        return cls(project_id=project_id, item_id=item_id, asset_name=asset_name)

datacosmos/stac/storage/storage_client.py

@@ -16,6 +16,7 @@ class StorageClient:
     def upload_item(
         self,
         item: DatacosmosItem,
+        project_id: str,
         assets_path: str | None = None,
         included_assets: list[str] | bool = True,
         max_workers: int = 4,
@@ -24,6 +25,7 @@ class StorageClient:
         """Proxy to Uploader.upload_item, without needing to pass client each call."""
         return self.uploader.upload_item(
             item=item,
+            project_id=project_id,
             assets_path=assets_path,
             included_assets=included_assets,
             max_workers=max_workers,

datacosmos/stac/storage/uploader.py

@@ -13,22 +13,37 @@ from datacosmos.stac.storage.storage_base import StorageBase
 
 
 class Uploader(StorageBase):
-    """Handles uploading files to Datacosmos storage and registering STAC items."""
+    """Upload a STAC item and its assets to Datacosmos storage, then register the item in the STAC API."""
 
     def __init__(self, client: DatacosmosClient):
-        """Handles uploading files to Datacosmos storage and registering STAC items."""
+        """Initialize the uploader.
+
+        Args:
+            client (DatacosmosClient): Pre-configured DatacosmosClient.
+        """
         super().__init__(client)
         self.item_client = ItemClient(client)
 
     def upload_item(
         self,
-        item: DatacosmosItem,
+        item: DatacosmosItem | str,
+        project_id: str,
         assets_path: str | None = None,
         included_assets: list[str] | bool = True,
         max_workers: int = 4,
         time_out: float = 60 * 60 * 1,
     ) -> DatacosmosItem:
-        """Upload a STAC item and its assets to Datacosmos."""
+        """Upload a STAC item (and optionally its assets) to Datacosmos.
+
+        `item` can be either:
+          • a DatacosmosItem instance, or
+          • the path to an item JSON file on disk.
+
+        If `included_assets` is:
+          • True  → upload every asset in the item
+          • list  → upload only the asset keys in that list
+          • False → upload nothing; just register the item
+        """
         if not assets_path and not isinstance(item, str):
             raise ValueError(
                 "assets_path must be provided if item is not the path to an item file."
@@ -37,8 +52,7 @@ class Uploader(StorageBase):
         if isinstance(item, str):
             item_filename = item
             item = self._load_item(item_filename)
-            if not assets_path:
-                assets_path = str(Path(item_filename).parent)
+            assets_path = assets_path or str(Path(item_filename).parent)
 
         assets_path = assets_path or str(Path.cwd())
 
@@ -50,18 +64,18 @@ class Uploader(StorageBase):
             else []
         )
 
-        jobs = [(item, asset_key, assets_path) for asset_key in upload_assets]
-
+        jobs = [
+            (item, asset_key, assets_path, project_id) for asset_key in upload_assets
+        ]
         self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
 
         self.item_client.add_item(item)
-
         return item
 
     def upload_from_file(
         self, src: str, dst: str, mime_type: str | None = None
     ) -> None:
-        """Uploads a single file to the specified destination path."""
+        """Upload a single file to the specified destination path in storage."""
         url = self.base_url.with_suffix(dst)
         mime = mime_type or self._guess_mime(src)
         headers = {"Content-Type": mime}
@@ -71,25 +85,40 @@ class Uploader(StorageBase):
 
     @staticmethod
     def _load_item(item_json_file_path: str) -> DatacosmosItem:
+        """Load a DatacosmosItem from a JSON file on disk."""
         with open(item_json_file_path, "rb") as file:
             data = file.read().decode("utf-8")
         return TypeAdapter(DatacosmosItem).validate_json(data)
 
     def _upload_asset(
-        self, item: DatacosmosItem, asset_key: str, assets_path: str
+        self, item: DatacosmosItem, asset_key: str, assets_path: str, project_id: str
     ) -> None:
+        """Upload a single asset file and update its href inside the item object.
+
+        Runs in parallel via _run_in_threads().
+        """
         asset = item.assets[asset_key]
-        upload_path = UploadPath.from_item_path(item, "", Path(asset.href).name)
+
+        # Build storage key: project/<project_id>/<item_id>/<asset_name>
+        upload_path = UploadPath.from_item_path(
+            item,
+            project_id,
+            Path(asset.href).name,
+        )
+
         local_src = Path(assets_path) / asset.href
         if local_src.exists():
             src = str(local_src)
             asset.href = f"file:///{upload_path}"
         else:
+            # fallback: try matching just the filename inside assets_path
             src = str(Path(assets_path) / Path(asset.href).name)
-        self._update_asset_href(asset)
+
+        self._update_asset_href(asset)  # turn href into public URL
         self.upload_from_file(src, str(upload_path), mime_type=asset.type)
 
     def _update_asset_href(self, asset: Asset) -> None:
+        """Convert the storage key to a public HTTPS URL."""
         try:
             url = self.client.config.datacosmos_public_cloud_storage.as_domain_url()
             new_href = url.with_base(asset.href)  # type: ignore