cloudnet-api-client 0.5.1__tar.gz → 0.7.0__tar.gz

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.7.0 – 2025-05-06
+
+- Validate checksum optionally
+- Make site optional
+
+## 0.6.0 – 2025-04-22
+
+- Fix datetime parsing for Python 3.10
+- Add Instrument and Model objects to metadata
+
 ## 0.5.1 – 2025-04-04
 
 - Raise if downloading failed after retries

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.5.1
+Version: 0.7.0
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -65,7 +65,7 @@ Parameters:
 
 | name                | type                        | default | example                                              |
 | ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id             | `str`                       |         | "hyytiala"                                           |
+| site_id             | `str` or `list[str]`        | `None`  | "hyytiala"                                           |
 | date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
 | date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
 | date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
@@ -158,6 +158,7 @@ Parameters:
 | output_directory  | `PathLike` or `str`                            |         |
 | concurrency_limit | `int`                                          | 5       |
 | progress          | `bool` or `None`                               | `None`  |
+| validate_checksum | `bool`                                         | `False` |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/README.md

@@ -38,7 +38,7 @@ Parameters:
 
 | name                | type                        | default | example                                              |
 | ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id             | `str`                       |         | "hyytiala"                                           |
+| site_id             | `str` or `list[str]`        | `None`  | "hyytiala"                                           |
 | date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
 | date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
 | date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
@@ -131,6 +131,7 @@ Parameters:
 | output_directory  | `PathLike` or `str`                            |         |
 | concurrency_limit | `int`                                          | 5       |
 | progress          | `bool` or `None`                               | `None`  |
+| validate_checksum | `bool`                                         | `False` |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/cloudnet_api_client/client.py

@@ -85,7 +85,7 @@ class APIClient:
 
     def metadata(
         self,
-        site_id: str,
+        site_id: QueryParam = None,
         date: DateParam = None,
         date_from: DateParam = None,
         date_to: DateParam = None,
@@ -108,6 +108,9 @@ class APIClient:
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
+
+        _check_params(params, ("showLegacy",))
+
         no_instrument = not instrument_id or instrument_pid
         if no_instrument and (not product and model_id):
             files_res = []
@@ -129,7 +132,7 @@ class APIClient:
 
     def raw_metadata(
         self,
-        site_id: str,
+        site_id: QueryParam = None,
         date: DateParam = None,
         date_from: DateParam = None,
         date_to: DateParam = None,
@@ -181,6 +184,9 @@ class APIClient:
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
+
+        _check_params(params)
+
         res = self._get_response("raw-model-files", params)
         return _build_raw_model_meta_objects(res)
 
@@ -190,9 +196,16 @@ class APIClient:
         output_directory: str | PathLike,
         concurrency_limit: int = 5,
         progress: bool | None = None,
+        validate_checksum: bool = False,
     ) -> list[Path]:
         return asyncio.run(
-            self.adownload(metadata, output_directory, concurrency_limit, progress)
+            self.adownload(
+                metadata,
+                output_directory,
+                concurrency_limit,
+                progress,
+                validate_checksum,
+            )
         )
 
     async def adownload(
@@ -201,6 +214,7 @@ class APIClient:
         output_directory: str | PathLike,
         concurrency_limit: int = 5,
         progress: bool | None = None,
+        validate_checksum: bool = False,
     ) -> list[Path]:
         disable_progress = not progress if progress is not None else None
         output_directory = Path(output_directory).resolve()
@@ -211,6 +225,7 @@ class APIClient:
             output_directory,
             concurrency_limit,
             disable_progress,
+            validate_checksum,
         )
 
     @staticmethod
@@ -270,13 +285,13 @@ def _add_date_params(
         msg = "Cannot use 'date' with 'date_from' and 'date_to'"
         raise ValueError(msg)
     if date is not None:
-        start, stop = _parse_date(date)
+        start, stop = _parse_date_param(date)
         params["dateFrom"] = start.isoformat()
         params["dateTo"] = stop.isoformat()
     if date_from is not None:
-        params["dateFrom"] = _parse_date(date_from)[0].isoformat()
+        params["dateFrom"] = _parse_date_param(date_from)[0].isoformat()
     if date_to is not None:
-        params["dateTo"] = _parse_date(date_to)[1].isoformat()
+        params["dateTo"] = _parse_date_param(date_to)[1].isoformat()
 
     if updated_at is not None and (
         updated_at_from is not None or updated_at_to is not None
@@ -284,16 +299,16 @@ def _add_date_params(
         msg = "Cannot use 'updated_at' with 'updated_at_from' and 'updated_at_to'"
         raise ValueError(msg)
     if updated_at is not None:
-        start, stop = _parse_datetime(updated_at)
+        start, stop = _parse_datetime_param(updated_at)
         params["updatedAtFrom"] = start.isoformat()
         params["updatedAtTo"] = stop.isoformat()
     if updated_at_from is not None:
-        params["updatedAtFrom"] = _parse_datetime(updated_at_from)[0].isoformat()
+        params["updatedAtFrom"] = _parse_datetime_param(updated_at_from)[0].isoformat()
     if updated_at_to is not None:
-        params["updatedAtTo"] = _parse_datetime(updated_at_to)[1].isoformat()
+        params["updatedAtTo"] = _parse_datetime_param(updated_at_to)[1].isoformat()
 
 
-def _parse_date(date: DateParam) -> tuple[datetime.date, datetime.date]:
+def _parse_date_param(date: DateParam) -> tuple[datetime.date, datetime.date]:
     if isinstance(date, datetime.date):
         return date, date
     error = ValueError(f"Invalid date format: {date}")
@@ -316,7 +331,9 @@ def _parse_date(date: DateParam) -> tuple[datetime.date, datetime.date]:
     raise error
 
 
-def _parse_datetime(dt: DateTimeParam) -> tuple[datetime.datetime, datetime.datetime]:
+def _parse_datetime_param(
+    dt: DateTimeParam,
+) -> tuple[datetime.datetime, datetime.datetime]:
     if isinstance(dt, datetime.datetime):
         return dt, dt
     if isinstance(dt, datetime.date):
@@ -376,19 +393,49 @@ CONVERTED = {"measurement_date", "created_at", "updated_at", "size", "uuid"}
 
 
 def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
-    field_names = {f.name for f in fields(ProductMetadata)} - CONVERTED - {"product"}
+    field_names = (
+        {f.name for f in fields(ProductMetadata)}
+        - CONVERTED
+        - {"product", "instrument", "model"}
+    )
     return [
         ProductMetadata(
             **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
             product=Product(
                 id=obj["product"]["id"],
                 human_readable_name=obj["product"]["humanReadableName"],
-                type=[obj["product"]["type"][1:-1]],
+                type=obj["product"]["type"],
                 experimental=obj["product"]["experimental"],
             ),
+            instrument=Instrument(
+                instrument_id=obj["instrument"]["instrumentId"],
+                model=obj["instrument"]["model"],
+                type=obj["instrument"]["type"],
+                uuid=uuid.UUID(obj["instrument"]["uuid"]),
+                pid=obj["instrument"]["pid"],
+                owners=obj["instrument"]["owners"],
+                serial_number=obj["instrument"]["serialNumber"],
+                name=obj["instrument"]["name"],
+            )
+            if "instrument" in obj and obj["instrument"] is not None
+            else None,
+            model=Model(
+                model_id=obj["model"]["id"],
+                name=obj["model"]["humanReadableName"],
+                optimum_order=obj["model"]["optimumOrder"],
+                source_model_id=obj["model"]["sourceModelId"],
+                forecast_start=obj["model"]["forecastStart"]
+                if obj["model"]["forecastStart"] is not None
+                else None,
+                forecast_end=obj["model"]["forecastEnd"]
+                if obj["model"]["forecastEnd"] is not None
+                else None,
+            )
+            if "model" in obj and obj["model"] is not None
+            else None,
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
-            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
-            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            created_at=_parse_datetime(obj["createdAt"]),
+            updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
         )
@@ -412,8 +459,8 @@ def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
                 name=obj["instrumentInfo"]["name"],
             ),
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
-            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
-            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            created_at=_parse_datetime(obj["createdAt"]),
+            updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
         )
@@ -439,8 +486,8 @@ def _build_raw_model_meta_objects(res: list[dict]) -> list[RawModelMetadata]:
                 else None,
             ),
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
-            created_at=datetime.datetime.fromisoformat(obj["createdAt"]),
-            updated_at=datetime.datetime.fromisoformat(obj["updatedAt"]),
+            created_at=_parse_datetime(obj["createdAt"]),
+            updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
         )
@@ -459,3 +506,12 @@ def _make_session() -> requests.Session:
     session.mount("https://", adapter)
     session.mount("http://", adapter)
     return session
+
+
+def _parse_datetime(dt: str) -> datetime.datetime:
+    return datetime.datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def _check_params(params: dict, ignore: tuple = ()) -> None:
+    if sum(1 for key, value in params.items() if key not in ignore and value) == 0:
+        raise TypeError("At least one of the parameters must be set.")

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/cloudnet_api_client/containers.py

@@ -83,3 +83,5 @@ class RawModelMetadata(Metadata):
 @dataclass(frozen=True, slots=True)
 class ProductMetadata(Metadata):
     product: Product
+    instrument: Instrument | None
+    model: Model | None

{cloudnet_api_client-0.5.1 → cloudnet_api_client-0.7.0}/cloudnet_api_client/dl.py

@@ -20,7 +20,9 @@ async def download_files(
     output_path: Path,
     concurrency_limit: int,
     disable_progress: bool | None,
+    validate_checksum: bool = False,
 ) -> list[Path]:
+    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
     semaphore = asyncio.Semaphore(concurrency_limit)
     full_paths = []
     async with aiohttp.ClientSession() as session:
@@ -29,7 +31,7 @@ async def download_files(
             download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
             destination = output_path / meta.download_url.split("/")[-1]
             full_paths.append(destination)
-            if destination.exists() and _file_checksum_matches(meta, destination):
+            if destination.exists() and file_exists(meta, destination):
                 logging.debug(f"Already downloaded: {destination}")
                 continue
             task = asyncio.create_task(
@@ -97,8 +99,17 @@ async def _download_file(
         logging.debug(f"Downloaded: {destination}")
 
 
-def _file_checksum_matches(
+def _checksum_matches(
     meta: ProductMetadata | RawMetadata | RawModelMetadata, destination: Path
 ) -> bool:
     fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
     return fun(destination) == meta.checksum
+
+
+def _size_and_name_matches(
+    meta: ProductMetadata | RawMetadata | RawModelMetadata, destination: Path
+) -> bool:
+    return (
+        destination.stat().st_size == meta.size
+        and destination.name == meta.download_url.split("/")[-1]
+    )

cloudnet_api_client-0.7.0/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.7.0"

cloudnet_api_client-0.5.1/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.5.1"