cloudnet-api-client 0.4.1__tar.gz → 0.5.1__tar.gz

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.5.1 – 2025-04-04
+
+- Raise if downloading failed after retries
+- Adjust logging
+
+## 0.5.0 – 2025-04-04
+
+- Add option to query one site
+- Add undocumented `raw_model_metadata` function
+
 ## 0.4.1 – 2025-04-03
 
 - Fix types

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.4.1
+Version: 0.5.1
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -41,9 +41,9 @@ python3 -m pip install cloudnet-api-client
 ## Quickstart
 
 ```python
-import cloudnet_api_client as cac
+from cloudnet_api_client import APIClient
 
-client = cac.APIClient()
+client = APIClient()
 
 sites = client.sites(type="cloudnet")
 products = client.products()
@@ -127,9 +127,10 @@ Fetch cloudnet sites.
 
 Parameters:
 
-| name | type                 | Choices                                   | default |
-| ---- | -------------------- | ----------------------------------------- | ------- |
-| type | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
+| name    | type                 | Choices                                   | default |
+| ------- | -------------------- | ----------------------------------------- | ------- |
+| site_id | `str`                |                                           | `None`  |
+| type    | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
 
 ### `APIClient().products()` &rarr; `list[Product]`
 

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/README.md

@@ -14,9 +14,9 @@ python3 -m pip install cloudnet-api-client
 ## Quickstart
 
 ```python
-import cloudnet_api_client as cac
+from cloudnet_api_client import APIClient
 
-client = cac.APIClient()
+client = APIClient()
 
 sites = client.sites(type="cloudnet")
 products = client.products()
@@ -100,9 +100,10 @@ Fetch cloudnet sites.
 
 Parameters:
 
-| name | type                 | Choices                                   | default |
-| ---- | -------------------- | ----------------------------------------- | ------- |
-| type | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
+| name    | type                 | Choices                                   | default |
+| ------- | -------------------- | ----------------------------------------- | ------- |
+| site_id | `str`                |                                           | `None`  |
+| type    | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
 
 ### `APIClient().products()` → `list[Product]`
 

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/cloudnet_api_client/client.py

@@ -19,16 +19,18 @@ from cloudnet_api_client.containers import (
     SITE_TYPE,
     STATUS,
     Instrument,
+    Model,
     Product,
     ProductMetadata,
     RawMetadata,
+    RawModelMetadata,
     Site,
 )
 from cloudnet_api_client.dl import download_files
 
 T = TypeVar("T")
-MetadataList = list[ProductMetadata] | list[RawMetadata]
-TMetadata = TypeVar("TMetadata", ProductMetadata, RawMetadata)
+MetadataList = list[ProductMetadata] | list[RawMetadata] | list[RawModelMetadata]
+TMetadata = TypeVar("TMetadata", ProductMetadata, RawMetadata, RawModelMetadata)
 DateParam = str | datetime.date | None
 DateTimeParam = str | datetime.datetime | datetime.date | None
 QueryParam = str | list[str] | None
@@ -43,9 +45,15 @@ class APIClient:
         self.base_url = base_url
         self.session = session or _make_session()
 
-    def sites(self, type: SITE_TYPE | list[SITE_TYPE] | None = None) -> list[Site]:
-        params = {"type": type}
-        res = self._get_response("sites", params)
+    def sites(
+        self,
+        site_id: str | None = None,
+        type: SITE_TYPE | list[SITE_TYPE] | None = None,
+    ) -> list[Site]:
+        if site_id:
+            res = self._get_response(f"sites/{site_id}")
+        else:
+            res = self._get_response("sites", {"type": type})
         return _build_objects(res, Site)
 
     def products(
@@ -148,6 +156,34 @@ class APIClient:
         res = self._get_response("raw-files", params)
         return _build_raw_meta_objects(res)
 
+    def raw_model_metadata(
+        self,
+        site_id: str,
+        model_id: QueryParam = None,
+        date: DateParam = None,
+        date_from: DateParam = None,
+        date_to: DateParam = None,
+        updated_at: DateTimeParam = None,
+        updated_at_from: DateTimeParam = None,
+        updated_at_to: DateTimeParam = None,
+        filename_prefix: QueryParam = None,
+        filename_suffix: QueryParam = None,
+        status: STATUS | list[STATUS] | None = None,
+    ) -> list[RawModelMetadata]:
+        """For internal CLU use only. Will change in the future."""
+        params = {
+            "site": site_id,
+            "filenamePrefix": filename_prefix,
+            "filenameSuffix": filename_suffix,
+            "status": status,
+            "model": model_id,
+        }
+        _add_date_params(
+            params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
+        )
+        res = self._get_response("raw-model-files", params)
+        return _build_raw_model_meta_objects(res)
+
     def download(
         self,
         metadata: MetadataList,
@@ -215,7 +251,10 @@ class APIClient:
         url = urljoin(self.base_url, endpoint)
         res = self.session.get(url, params=params, timeout=120)
         res.raise_for_status()
-        return res.json()
+        data = res.json()
+        if isinstance(data, dict):
+            data = [data]
+        return data
 
 
 def _add_date_params(
@@ -382,6 +421,33 @@ def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
     ]
 
 
+def _build_raw_model_meta_objects(res: list[dict]) -> list[RawModelMetadata]:
+    field_names = {f.name for f in fields(RawModelMetadata)} - CONVERTED - {"model"}
+    return [
+        RawModelMetadata(
+            **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
+            model=Model(
+                model_id=obj["model"]["id"],
+                name=obj["model"]["humanReadableName"],
+                optimum_order=int(obj["model"]["optimumOrder"]),
+                source_model_id=obj["model"]["sourceModelId"],
+                forecast_start=int(obj["model"]["forecastStart"])
+                if obj["model"]["forecastStart"] is not None
+                else None,
+                forecast_end=int(obj["model"]["forecastEnd"])
+                if obj["model"]["forecastEnd"] is not None
+                else None,
+            ),
+            measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
+            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
+            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            size=int(obj["size"]),
+            uuid=uuid.UUID(obj["uuid"]),
+        )
+        for obj in res
+    ]
+
+
 def _to_snake(name: str) -> str:
     return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
 

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/cloudnet_api_client/containers.py

@@ -22,7 +22,6 @@ class Site:
     country_code: str
     country_subdivision_code: str | None
     type: list[SITE_TYPE]
-    status: Literal["active", "inactive"]
     gaw: str | None
 
 
@@ -46,6 +45,16 @@ class Instrument:
     serial_number: str | None
 
 
+@dataclass(frozen=True, slots=True)
+class Model:
+    model_id: str
+    name: str
+    optimum_order: int
+    source_model_id: str
+    forecast_start: int | None
+    forecast_end: int | None
+
+
 @dataclass(frozen=True, slots=True)
 class Metadata:
     uuid: uuid.UUID
@@ -65,6 +74,12 @@ class RawMetadata(Metadata):
     tags: list[str] | None
 
 
+@dataclass(frozen=True, slots=True)
+class RawModelMetadata(Metadata):
+    status: STATUS
+    model: Model
+
+
 @dataclass(frozen=True, slots=True)
 class ProductMetadata(Metadata):
     product: Product

{cloudnet_api_client-0.4.1 → cloudnet_api_client-0.5.1}/cloudnet_api_client/dl.py

@@ -7,12 +7,16 @@ from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
 
 from cloudnet_api_client import utils
-from cloudnet_api_client.containers import ProductMetadata, RawMetadata
+from cloudnet_api_client.containers import (
+    ProductMetadata,
+    RawMetadata,
+    RawModelMetadata,
+)
 
 
 async def download_files(
     base_url: str,
-    metadata: list[ProductMetadata] | list[RawMetadata],
+    metadata: list[ProductMetadata] | list[RawMetadata] | list[RawModelMetadata],
     output_path: Path,
     concurrency_limit: int,
     disable_progress: bool | None,
@@ -26,7 +30,7 @@ async def download_files(
             destination = output_path / meta.download_url.split("/")[-1]
             full_paths.append(destination)
             if destination.exists() and _file_checksum_matches(meta, destination):
-                logging.info(f"Already downloaded: {destination}")
+                logging.debug(f"Already downloaded: {destination}")
                 continue
             task = asyncio.create_task(
                 _download_file_with_retries(
@@ -57,6 +61,7 @@ async def _download_file_with_retries(
             logging.warning(f"Attempt {attempt} failed for {url}: {e}")
             if attempt == max_retries:
                 logging.error(f"Giving up on {url} after {max_retries} attempts.")
+                raise e
             else:
                 # Exponential backoff before retrying
                 await asyncio.sleep(2**attempt)
@@ -89,11 +94,11 @@ async def _download_file(
                         break
                     file_out.write(chunk)
                     bar.update(len(chunk))
-        logging.info(f"Downloaded: {destination}")
+        logging.debug(f"Downloaded: {destination}")
 
 
 def _file_checksum_matches(
-    meta: ProductMetadata | RawMetadata, destination: Path
+    meta: ProductMetadata | RawMetadata | RawModelMetadata, destination: Path
 ) -> bool:
-    fun = utils.md5sum if isinstance(meta, RawMetadata) else utils.sha256sum
+    fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
     return fun(destination) == meta.checksum

cloudnet_api_client-0.5.1/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"

cloudnet_api_client-0.4.1/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.4.1"