cloudnet-api-client 0.6.0__tar.gz → 0.8.0__tar.gz

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.8.0 – 2025-05-14
+
+- Add site to metadata responses
+
+## 0.7.0 – 2025-05-06
+
+- Validate checksum optionally
+- Make site optional
+
 ## 0.6.0 – 2025-04-22
 
 - Fix datetime parsing for Python 3.10

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.6.0
+Version: 0.8.0
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -65,7 +65,7 @@ Parameters:
 
 | name                | type                        | default | example                                              |
 | ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id             | `str`                       |         | "hyytiala"                                           |
+| site_id             | `str` or `list[str]`        | `None`  | "hyytiala"                                           |
 | date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
 | date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
 | date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
@@ -158,6 +158,7 @@ Parameters:
 | output_directory  | `PathLike` or `str`                            |         |
 | concurrency_limit | `int`                                          | 5       |
 | progress          | `bool` or `None`                               | `None`  |
+| validate_checksum | `bool`                                         | `False` |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/README.md

@@ -38,7 +38,7 @@ Parameters:
 
 | name                | type                        | default | example                                              |
 | ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id             | `str`                       |         | "hyytiala"                                           |
+| site_id             | `str` or `list[str]`        | `None`  | "hyytiala"                                           |
 | date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
 | date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
 | date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
@@ -131,6 +131,7 @@ Parameters:
 | output_directory  | `PathLike` or `str`                            |         |
 | concurrency_limit | `int`                                          | 5       |
 | progress          | `bool` or `None`                               | `None`  |
+| validate_checksum | `bool`                                         | `False` |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/cloudnet_api_client/client.py

@@ -85,7 +85,7 @@ class APIClient:
 
     def metadata(
         self,
-        site_id: str,
+        site_id: QueryParam = None,
         date: DateParam = None,
         date_from: DateParam = None,
         date_to: DateParam = None,
@@ -108,6 +108,9 @@ class APIClient:
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
+
+        _check_params(params, ("showLegacy",))
+
         no_instrument = not instrument_id or instrument_pid
         if no_instrument and (not product and model_id):
             files_res = []
@@ -129,7 +132,7 @@ class APIClient:
 
     def raw_metadata(
         self,
-        site_id: str,
+        site_id: QueryParam = None,
         date: DateParam = None,
         date_from: DateParam = None,
         date_to: DateParam = None,
@@ -158,7 +161,7 @@ class APIClient:
 
     def raw_model_metadata(
         self,
-        site_id: str,
+        site_id: QueryParam = None,
         model_id: QueryParam = None,
         date: DateParam = None,
         date_from: DateParam = None,
@@ -181,6 +184,9 @@ class APIClient:
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
+
+        _check_params(params)
+
         res = self._get_response("raw-model-files", params)
         return _build_raw_model_meta_objects(res)
 
@@ -190,9 +196,16 @@ class APIClient:
         output_directory: str | PathLike,
         concurrency_limit: int = 5,
         progress: bool | None = None,
+        validate_checksum: bool = False,
     ) -> list[Path]:
         return asyncio.run(
-            self.adownload(metadata, output_directory, concurrency_limit, progress)
+            self.adownload(
+                metadata,
+                output_directory,
+                concurrency_limit,
+                progress,
+                validate_checksum,
+            )
         )
 
     async def adownload(
@@ -201,6 +214,7 @@ class APIClient:
         output_directory: str | PathLike,
         concurrency_limit: int = 5,
         progress: bool | None = None,
+        validate_checksum: bool = False,
     ) -> list[Path]:
         disable_progress = not progress if progress is not None else None
         output_directory = Path(output_directory).resolve()
@@ -211,6 +225,7 @@ class APIClient:
             output_directory,
             concurrency_limit,
             disable_progress,
+            validate_checksum,
         )
 
     @staticmethod
@@ -381,7 +396,7 @@ def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
     field_names = (
         {f.name for f in fields(ProductMetadata)}
         - CONVERTED
-        - {"product", "instrument", "model"}
+        - {"product", "instrument", "model", "site"}
     )
     return [
         ProductMetadata(
@@ -392,30 +407,10 @@ def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
                 type=obj["product"]["type"],
                 experimental=obj["product"]["experimental"],
             ),
-            instrument=Instrument(
-                instrument_id=obj["instrument"]["instrumentId"],
-                model=obj["instrument"]["model"],
-                type=obj["instrument"]["type"],
-                uuid=uuid.UUID(obj["instrument"]["uuid"]),
-                pid=obj["instrument"]["pid"],
-                owners=obj["instrument"]["owners"],
-                serial_number=obj["instrument"]["serialNumber"],
-                name=obj["instrument"]["name"],
-            )
+            instrument=_create_instrument_object(obj["instrument"])
             if "instrument" in obj and obj["instrument"] is not None
             else None,
-            model=Model(
-                model_id=obj["model"]["id"],
-                name=obj["model"]["humanReadableName"],
-                optimum_order=obj["model"]["optimumOrder"],
-                source_model_id=obj["model"]["sourceModelId"],
-                forecast_start=obj["model"]["forecastStart"]
-                if obj["model"]["forecastStart"] is not None
-                else None,
-                forecast_end=obj["model"]["forecastEnd"]
-                if obj["model"]["forecastEnd"] is not None
-                else None,
-            )
+            model=_create_model_object(obj["model"])
             if "model" in obj and obj["model"] is not None
             else None,
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
@@ -423,63 +418,96 @@ def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
+            site=_create_site_object(obj["site"]),
         )
         for obj in res
     ]
 
 
 def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
-    field_names = {f.name for f in fields(RawMetadata)} - CONVERTED - {"instrument"}
+    field_names = (
+        {f.name for f in fields(RawMetadata)} - CONVERTED - {"instrument", "site"}
+    )
     return [
         RawMetadata(
             **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
-            instrument=Instrument(
-                instrument_id=obj["instrumentInfo"]["instrumentId"],
-                model=obj["instrumentInfo"]["model"],
-                type=obj["instrumentInfo"]["type"],
-                uuid=uuid.UUID(obj["instrumentInfo"]["uuid"]),
-                pid=obj["instrumentInfo"]["pid"],
-                owners=obj["instrumentInfo"]["owners"],
-                serial_number=obj["instrumentInfo"]["serialNumber"],
-                name=obj["instrumentInfo"]["name"],
-            ),
+            instrument=_create_instrument_object(obj["instrumentInfo"]),
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
             created_at=_parse_datetime(obj["createdAt"]),
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
+            site=_create_site_object(obj["site"]),
         )
         for obj in res
     ]
 
 
 def _build_raw_model_meta_objects(res: list[dict]) -> list[RawModelMetadata]:
-    field_names = {f.name for f in fields(RawModelMetadata)} - CONVERTED - {"model"}
+    field_names = (
+        {f.name for f in fields(RawModelMetadata)} - CONVERTED - {"model", "site"}
+    )
     return [
         RawModelMetadata(
             **{_to_snake(k): v for k, v in obj.items() if _to_snake(k) in field_names},
-            model=Model(
-                model_id=obj["model"]["id"],
-                name=obj["model"]["humanReadableName"],
-                optimum_order=int(obj["model"]["optimumOrder"]),
-                source_model_id=obj["model"]["sourceModelId"],
-                forecast_start=int(obj["model"]["forecastStart"])
-                if obj["model"]["forecastStart"] is not None
-                else None,
-                forecast_end=int(obj["model"]["forecastEnd"])
-                if obj["model"]["forecastEnd"] is not None
-                else None,
-            ),
+            model=_create_model_object(obj["model"]),
             measurement_date=datetime.date.fromisoformat(obj["measurementDate"]),
             created_at=_parse_datetime(obj["createdAt"]),
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
             uuid=uuid.UUID(obj["uuid"]),
+            site=_create_site_object(obj["site"]),
         )
         for obj in res
     ]
 
 
+def _create_model_object(metadata: dict) -> Model:
+    return Model(
+        model_id=metadata["id"],
+        name=metadata["humanReadableName"],
+        optimum_order=int(metadata["optimumOrder"]),
+        source_model_id=metadata["sourceModelId"],
+        forecast_start=int(metadata["forecastStart"])
+        if metadata["forecastStart"] is not None
+        else None,
+        forecast_end=int(metadata["forecastEnd"])
+        if metadata["forecastEnd"] is not None
+        else None,
+    )
+
+
+def _create_site_object(metadata: dict) -> Site:
+    return Site(
+        id=metadata["id"],
+        human_readable_name=metadata["humanReadableName"],
+        station_name=metadata["stationName"],
+        latitude=float(metadata["latitude"]),
+        longitude=float(metadata["longitude"]),
+        altitude=float(metadata["altitude"]),
+        dvas_id=metadata["dvasId"],
+        actris_id=metadata["actrisId"],
+        country=metadata["country"],
+        country_code=metadata["countryCode"],
+        country_subdivision_code=metadata["countrySubdivisionCode"],
+        type=metadata["type"],
+        gaw=metadata["gaw"],
+    )
+
+
+def _create_instrument_object(metadata: dict) -> Instrument:
+    return Instrument(
+        instrument_id=metadata["instrumentId"],
+        model=metadata["model"],
+        type=metadata["type"],
+        uuid=uuid.UUID(metadata["uuid"]),
+        pid=metadata["pid"],
+        owners=metadata["owners"],
+        serial_number=metadata["serialNumber"],
+        name=metadata["name"],
+    )
+
+
 def _to_snake(name: str) -> str:
     return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
 
@@ -495,3 +523,8 @@ def _make_session() -> requests.Session:
 
 def _parse_datetime(dt: str) -> datetime.datetime:
     return datetime.datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def _check_params(params: dict, ignore: tuple = ()) -> None:
+    if sum(1 for key, value in params.items() if key not in ignore and value) == 0:
+        raise TypeError("At least one of the parameters must be set.")

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/cloudnet_api_client/containers.py

@@ -65,6 +65,7 @@ class Metadata:
     measurement_date: datetime.date
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    site: Site
 
 
 @dataclass(frozen=True, slots=True)

{cloudnet_api_client-0.6.0 → cloudnet_api_client-0.8.0}/cloudnet_api_client/dl.py

@@ -20,7 +20,9 @@ async def download_files(
     output_path: Path,
     concurrency_limit: int,
     disable_progress: bool | None,
+    validate_checksum: bool = False,
 ) -> list[Path]:
+    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
     semaphore = asyncio.Semaphore(concurrency_limit)
     full_paths = []
     async with aiohttp.ClientSession() as session:
@@ -29,7 +31,7 @@ async def download_files(
             download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
             destination = output_path / meta.download_url.split("/")[-1]
             full_paths.append(destination)
-            if destination.exists() and _file_checksum_matches(meta, destination):
+            if destination.exists() and file_exists(meta, destination):
                 logging.debug(f"Already downloaded: {destination}")
                 continue
             task = asyncio.create_task(
@@ -97,8 +99,17 @@ async def _download_file(
         logging.debug(f"Downloaded: {destination}")
 
 
-def _file_checksum_matches(
+def _checksum_matches(
     meta: ProductMetadata | RawMetadata | RawModelMetadata, destination: Path
 ) -> bool:
     fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
     return fun(destination) == meta.checksum
+
+
+def _size_and_name_matches(
+    meta: ProductMetadata | RawMetadata | RawModelMetadata, destination: Path
+) -> bool:
+    return (
+        destination.stat().st_size == meta.size
+        and destination.name == meta.download_url.split("/")[-1]
+    )

cloudnet_api_client-0.8.0/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.8.0"

cloudnet_api_client-0.6.0/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.6.0"