cloudnet-api-client 0.3.0__tar.gz → 0.4.1__tar.gz

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.4.1 – 2025-04-03
+
+- Fix types
+
+## 0.4.0 – 2025-04-03
+
+- Move download function to APIClient class
+- Add status parameter
+
 ## 0.3.0 – 2025-04-02
 
 - Move `filename_prefix` and `filename_suffix` parameters

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.3.0
+Version: 0.4.1
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -49,10 +49,10 @@ sites = client.sites(type="cloudnet")
 products = client.products()
 
 metadata = client.metadata("hyytiala", "2021-01-01", product=["mwr", "radar"])
-cac.download(metadata, "data/")
+client.download(metadata, "data/")
 
 raw_metadata = client.raw_metadata("granada", date="2024-01", instrument_id="parsivel")
-cac.download(raw_metadata, "data_raw/")
+client.download(raw_metadata, "data_raw/")
 ```
 
 ## Documentation
@@ -78,6 +78,7 @@ Parameters:
 | show_legacy\*       | `bool`                      | `False` |                                                      |
 | filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
 | filename_suffix\*\* | `str` or `list[str]`        | `None`  | ".lv1"                                               |
+| status\*\*          | `str` or `list[str]`        | `None`  | "created", "uploaded", "processed" or "invalid"      |
 
 \* = only in `metadata()`
 
@@ -110,13 +111,15 @@ Additional filtering of fetched metadata.
 
 Parameters:
 
-| name               | type                                           | default |
-| ------------------ | ---------------------------------------------- | ------- |
-| metadata           | `list[RawMetadata]` or `list[ProductMetadata]` |         |
-| include_pattern    | `str`                                          | `None`  |
-| exclude_pattern    | `str`                                          | `None`  |
-| include_tag_subset | `set[str]`                                     | `None`  |
-| exclude_tag_subset | `set[str]`                                     | `None`  |
+| name                 | type                                           | default |
+| -------------------- | ---------------------------------------------- | ------- |
+| metadata             | `list[RawMetadata]` or `list[ProductMetadata]` |         |
+| include_pattern      | `str`                                          | `None`  |
+| exclude_pattern      | `str`                                          | `None`  |
+| include_tag_subset\* | `set[str]`                                     | `None`  |
+| exclude_tag_subset\* | `set[str]`                                     | `None`  |
+
+\* = only with `RawMetadata`
 
 ### `APIClient().sites()` &rarr; `list[Site]`
 
@@ -142,7 +145,7 @@ Parameters:
 
 Fetch cloudnet instruments.
 
-### `cloudnet_api_client.download(list[Metadata])` &rarr; `list[Path]`
+### `APIClient().download(list[Metadata])` &rarr; `list[Path]`
 
 Download files from the fetched metadata.
 

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/README.md

@@ -22,10 +22,10 @@ sites = client.sites(type="cloudnet")
 products = client.products()
 
 metadata = client.metadata("hyytiala", "2021-01-01", product=["mwr", "radar"])
-cac.download(metadata, "data/")
+client.download(metadata, "data/")
 
 raw_metadata = client.raw_metadata("granada", date="2024-01", instrument_id="parsivel")
-cac.download(raw_metadata, "data_raw/")
+client.download(raw_metadata, "data_raw/")
 ```
 
 ## Documentation
@@ -51,6 +51,7 @@ Parameters:
 | show_legacy\*       | `bool`                      | `False` |                                                      |
 | filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
 | filename_suffix\*\* | `str` or `list[str]`        | `None`  | ".lv1"                                               |
+| status\*\*          | `str` or `list[str]`        | `None`  | "created", "uploaded", "processed" or "invalid"      |
 
 \* = only in `metadata()`
 
@@ -83,13 +84,15 @@ Additional filtering of fetched metadata.
 
 Parameters:
 
-| name               | type                                           | default |
-| ------------------ | ---------------------------------------------- | ------- |
-| metadata           | `list[RawMetadata]` or `list[ProductMetadata]` |         |
-| include_pattern    | `str`                                          | `None`  |
-| exclude_pattern    | `str`                                          | `None`  |
-| include_tag_subset | `set[str]`                                     | `None`  |
-| exclude_tag_subset | `set[str]`                                     | `None`  |
+| name                 | type                                           | default |
+| -------------------- | ---------------------------------------------- | ------- |
+| metadata             | `list[RawMetadata]` or `list[ProductMetadata]` |         |
+| include_pattern      | `str`                                          | `None`  |
+| exclude_pattern      | `str`                                          | `None`  |
+| include_tag_subset\* | `set[str]`                                     | `None`  |
+| exclude_tag_subset\* | `set[str]`                                     | `None`  |
+
+\* = only with `RawMetadata`
 
 ### `APIClient().sites()` → `list[Site]`
 
@@ -115,7 +118,7 @@ Parameters:
 
 Fetch cloudnet instruments.
 
-### `cloudnet_api_client.download(list[Metadata])` → `list[Path]`
+### `APIClient().download(list[Metadata])` → `list[Path]`
 
 Download files from the fetched metadata.
 

cloudnet_api_client-0.4.1/cloudnet_api_client/__init__.py

@@ -0,0 +1 @@
+from .client import APIClient as APIClient

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/cloudnet_api_client/client.py

@@ -1,7 +1,12 @@
+import asyncio
 import calendar
 import datetime
+import os
 import re
+import uuid
 from dataclasses import fields, is_dataclass
+from os import PathLike
+from pathlib import Path
 from typing import TypeVar, cast
 from urllib.parse import urljoin
 
@@ -12,15 +17,18 @@ from urllib3.util.retry import Retry
 from cloudnet_api_client.containers import (
     PRODUCT_TYPE,
     SITE_TYPE,
+    STATUS,
     Instrument,
-    Metadata,
     Product,
     ProductMetadata,
     RawMetadata,
     Site,
 )
+from cloudnet_api_client.dl import download_files
 
 T = TypeVar("T")
+MetadataList = list[ProductMetadata] | list[RawMetadata]
+TMetadata = TypeVar("TMetadata", ProductMetadata, RawMetadata)
 DateParam = str | datetime.date | None
 DateTimeParam = str | datetime.datetime | datetime.date | None
 QueryParam = str | list[str] | None
@@ -58,7 +66,7 @@ class APIClient:
                 instrument_id=obj["instrument"]["id"],
                 model=obj["model"],
                 type=obj["type"],
-                uuid=obj["uuid"],
+                uuid=uuid.UUID(obj["uuid"]),
                 pid=obj["pid"],
                 owners=obj["owners"],
                 serial_number=obj["serialNumber"],
@@ -124,6 +132,7 @@ class APIClient:
         instrument_pid: QueryParam = None,
         filename_prefix: QueryParam = None,
         filename_suffix: QueryParam = None,
+        status: STATUS | list[STATUS] | None = None,
     ) -> list[RawMetadata]:
         params = {
             "site": site_id,
@@ -131,6 +140,7 @@ class APIClient:
             "instrumentPid": instrument_pid,
             "filenamePrefix": filename_prefix,
             "filenameSuffix": filename_suffix,
+            "status": status,
         }
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
@@ -138,14 +148,43 @@ class APIClient:
         res = self._get_response("raw-files", params)
         return _build_raw_meta_objects(res)
 
+    def download(
+        self,
+        metadata: MetadataList,
+        output_directory: str | PathLike,
+        concurrency_limit: int = 5,
+        progress: bool | None = None,
+    ) -> list[Path]:
+        return asyncio.run(
+            self.adownload(metadata, output_directory, concurrency_limit, progress)
+        )
+
+    async def adownload(
+        self,
+        metadata: MetadataList,
+        output_directory: str | PathLike,
+        concurrency_limit: int = 5,
+        progress: bool | None = None,
+    ) -> list[Path]:
+        disable_progress = not progress if progress is not None else None
+        output_directory = Path(output_directory).resolve()
+        os.makedirs(output_directory, exist_ok=True)
+        return await download_files(
+            self.base_url,
+            metadata,
+            output_directory,
+            concurrency_limit,
+            disable_progress,
+        )
+
     @staticmethod
     def filter(
-        metadata: list[Metadata],
+        metadata: list[TMetadata],
         include_pattern: str | None = None,
         exclude_pattern: str | None = None,
         include_tag_subset: set[str] | None = None,
         exclude_tag_subset: set[str] | None = None,
-    ) -> list[Metadata]:
+    ) -> list[TMetadata]:
         if include_pattern:
             metadata = [
                 m for m in metadata if re.search(include_pattern, m.filename, re.I)
@@ -294,8 +333,11 @@ def _build_objects(res: list[dict], object_type: type[T]) -> list[T]:
     return cast(list[T], objects)
 
 
+CONVERTED = {"measurement_date", "created_at", "updated_at", "size", "uuid"}
+
+
 def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
-    field_names = {f.name for f in fields(ProductMetadata)} - {"product"}
+    field_names = {f.name for f in fields(ProductMetadata)} - CONVERTED - {"product"}
     return [
         ProductMetadata(
             **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
@@ -305,13 +347,18 @@ def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
                 type=[obj["product"]["type"][1:-1]],
                 experimental=obj["product"]["experimental"],
             ),
+            measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
+            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
+            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            size=int(obj["size"]),
+            uuid=uuid.UUID(obj["uuid"]),
         )
         for obj in res
     ]
 
 
 def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
-    field_names = {f.name for f in fields(RawMetadata)} - {"instrument"}
+    field_names = {f.name for f in fields(RawMetadata)} - CONVERTED - {"instrument"}
     return [
         RawMetadata(
             **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
@@ -319,12 +366,17 @@ def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
                 instrument_id=obj["instrumentInfo"]["instrumentId"],
                 model=obj["instrumentInfo"]["model"],
                 type=obj["instrumentInfo"]["type"],
-                uuid=obj["instrumentInfo"]["uuid"],
+                uuid=uuid.UUID(obj["instrumentInfo"]["uuid"]),
                 pid=obj["instrumentInfo"]["pid"],
                 owners=obj["instrumentInfo"]["owners"],
                 serial_number=obj["instrumentInfo"]["serialNumber"],
                 name=obj["instrumentInfo"]["name"],
             ),
+            measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
+            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
+            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            size=int(obj["size"]),
+            uuid=uuid.UUID(obj["uuid"]),
         )
         for obj in res
     ]

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/cloudnet_api_client/containers.py

@@ -1,9 +1,11 @@
+import datetime
 import uuid
 from dataclasses import dataclass
 from typing import Literal
 
 SITE_TYPE = Literal["cloudnet", "model", "hidden", "campaign"]
 PRODUCT_TYPE = Literal["instrument", "geophysical", "evaluation", "model"]
+STATUS = Literal["created", "uploaded", "processed", "invalid"]
 
 
 @dataclass(frozen=True, slots=True)
@@ -50,15 +52,15 @@ class Metadata:
     checksum: str
     size: int
     filename: str
-    measurement_date: str
     download_url: str
-    created_at: str
-    updated_at: str
+    measurement_date: datetime.date
+    created_at: datetime.datetime
+    updated_at: datetime.datetime
 
 
 @dataclass(frozen=True, slots=True)
 class RawMetadata(Metadata):
-    status: Literal["created", "uploaded", "processed", "invalid"]
+    status: STATUS
     instrument: Instrument
     tags: list[str] | None
 

{cloudnet_api_client-0.3.0 → cloudnet_api_client-0.4.1}/cloudnet_api_client/dl.py

@@ -1,7 +1,5 @@
 import asyncio
 import logging
-import os
-from os import PathLike
 from pathlib import Path
 
 import aiohttp
@@ -11,36 +9,10 @@ from tqdm.asyncio import tqdm_asyncio
 from cloudnet_api_client import utils
 from cloudnet_api_client.containers import ProductMetadata, RawMetadata
 
-MetadataList = list[ProductMetadata] | list[RawMetadata]
 
-
-def download(
-    metadata: MetadataList,
-    output_directory: str | PathLike,
-    concurrency_limit: int = 5,
-    progress: bool | None = None,
-) -> list[Path]:
-    return asyncio.run(
-        adownload(metadata, output_directory, concurrency_limit, progress)
-    )
-
-
-async def adownload(
-    metadata: MetadataList,
-    output_directory: str | PathLike,
-    concurrency_limit: int = 5,
-    progress: bool | None = None,
-) -> list[Path]:
-    disable_progress = not progress if progress is not None else None
-    output_directory = Path(output_directory).resolve()
-    os.makedirs(output_directory, exist_ok=True)
-    return await _download_files(
-        metadata, output_directory, concurrency_limit, disable_progress
-    )
-
-
-async def _download_files(
-    metadata: MetadataList,
+async def download_files(
+    base_url: str,
+    metadata: list[ProductMetadata] | list[RawMetadata],
     output_path: Path,
     concurrency_limit: int,
     disable_progress: bool | None,
@@ -50,6 +22,7 @@ async def _download_files(
     async with aiohttp.ClientSession() as session:
         tasks = []
         for meta in metadata:
+            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
             destination = output_path / meta.download_url.split("/")[-1]
             full_paths.append(destination)
             if destination.exists() and _file_checksum_matches(meta, destination):
@@ -57,7 +30,7 @@ async def _download_files(
                 continue
             task = asyncio.create_task(
                 _download_file_with_retries(
-                    session, meta.download_url, destination, semaphore, disable_progress
+                    session, download_url, destination, semaphore, disable_progress
                 )
             )
             tasks.append(task)

cloudnet_api_client-0.4.1/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.4.1"

cloudnet_api_client-0.3.0/cloudnet_api_client/__init__.py

@@ -1,3 +0,0 @@
-from .client import APIClient as APIClient
-from .dl import adownload as adownload
-from .dl import download as download

cloudnet_api_client-0.3.0/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"