cloudnet-api-client 0.2.1__tar.gz → 0.4.0__tar.gz

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.4.0 – 2025-04-03
+
+- Move download function to APIClient class
+- Add status parameter
+
+## 0.3.0 – 2025-04-02
+
+- Move `filename_prefix` and `filename_suffix` parameters
+- Fix query logic
+
 ## 0.2.1 – 2025-04-02
 
 - Return full paths of downloaded files

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.2.1
+Version: 0.4.0
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -49,10 +49,10 @@ sites = client.sites(type="cloudnet")
 products = client.products()
 
 metadata = client.metadata("hyytiala", "2021-01-01", product=["mwr", "radar"])
-cac.download(metadata, "data/")
+client.download(metadata, "data/")
 
 raw_metadata = client.raw_metadata("granada", date="2024-01", instrument_id="parsivel")
-cac.download(raw_metadata, "data_raw/")
+client.download(raw_metadata, "data_raw/")
 ```
 
 ## Documentation
@@ -63,22 +63,27 @@ Fetch product and raw file metadata from the Cloudnet data portal.
 
 Parameters:
 
-| name            | type                        | default | example                                              |
-| --------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id         | `str`                       |         | "hyytiala"                                           |
-| date            | `str` or `date`             | `None`  | "2024-01-01"                                         |
-| date_from       | `str` or `date`             | `None`  | "2025-01-01"                                         |
-| date_to         | `str` or `date`             | `None`  | "2025-01-01"                                         |
-| updated_at      | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| updated_at_from | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| updated_at_to   | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| instrument_id   | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
-| instrument_pid  | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
-| product\*       | `str` or `list[str]`        | `None`  | "classification"                                     |
-| show_legacy\*   | `bool`                      | `False` |                                                      |
+| name                | type                        | default | example                                              |
+| ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
+| site_id             | `str`                       |         | "hyytiala"                                           |
+| date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
+| date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
+| date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
+| updated_at          | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| updated_at_from     | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| updated_at_to       | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| instrument_id       | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
+| instrument_pid      | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
+| product\*           | `str` or `list[str]`        | `None`  | "classification"                                     |
+| show_legacy\*       | `bool`                      | `False` |                                                      |
+| filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
+| filename_suffix\*\* | `str` or `list[str]`        | `None`  | ".lv1"                                               |
+| status\*\*          | `str` or `list[str]`        | `None`  | "created", "uploaded", "processed" or "invalid"      |
 
 \* = only in `metadata()`
 
+\*\* = only in `raw_metadata()`
+
 **Date Handling**
 
 The `date`, `date_from` and `date_to` parameters support:
@@ -111,8 +116,6 @@ Parameters:
 | metadata           | `list[RawMetadata]` or `list[ProductMetadata]` |         |
 | include_pattern    | `str`                                          | `None`  |
 | exclude_pattern    | `str`                                          | `None`  |
-| filename_prefix    | `str`                                          | `None`  |
-| filename_suffix    | `str`                                          | `None`  |
 | include_tag_subset | `set[str]`                                     | `None`  |
 | exclude_tag_subset | `set[str]`                                     | `None`  |
 
@@ -140,7 +143,7 @@ Parameters:
 
 Fetch cloudnet instruments.
 
-### `cloudnet_api_client.download(list[Metadata])` &rarr; `list[Path]`
+### `APIClient().download(list[Metadata])` &rarr; `list[Path]`
 
 Download files from the fetched metadata.
 

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/README.md

@@ -22,10 +22,10 @@ sites = client.sites(type="cloudnet")
 products = client.products()
 
 metadata = client.metadata("hyytiala", "2021-01-01", product=["mwr", "radar"])
-cac.download(metadata, "data/")
+client.download(metadata, "data/")
 
 raw_metadata = client.raw_metadata("granada", date="2024-01", instrument_id="parsivel")
-cac.download(raw_metadata, "data_raw/")
+client.download(raw_metadata, "data_raw/")
 ```
 
 ## Documentation
@@ -36,22 +36,27 @@ Fetch product and raw file metadata from the Cloudnet data portal.
 
 Parameters:
 
-| name            | type                        | default | example                                              |
-| --------------- | --------------------------- | ------- | ---------------------------------------------------- |
-| site_id         | `str`                       |         | "hyytiala"                                           |
-| date            | `str` or `date`             | `None`  | "2024-01-01"                                         |
-| date_from       | `str` or `date`             | `None`  | "2025-01-01"                                         |
-| date_to         | `str` or `date`             | `None`  | "2025-01-01"                                         |
-| updated_at      | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| updated_at_from | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| updated_at_to   | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
-| instrument_id   | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
-| instrument_pid  | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
-| product\*       | `str` or `list[str]`        | `None`  | "classification"                                     |
-| show_legacy\*   | `bool`                      | `False` |                                                      |
+| name                | type                        | default | example                                              |
+| ------------------- | --------------------------- | ------- | ---------------------------------------------------- |
+| site_id             | `str`                       |         | "hyytiala"                                           |
+| date                | `str` or `date`             | `None`  | "2024-01-01"                                         |
+| date_from           | `str` or `date`             | `None`  | "2025-01-01"                                         |
+| date_to             | `str` or `date`             | `None`  | "2025-01-01"                                         |
+| updated_at          | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| updated_at_from     | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| updated_at_to       | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
+| instrument_id       | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
+| instrument_pid      | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
+| product\*           | `str` or `list[str]`        | `None`  | "classification"                                     |
+| show_legacy\*       | `bool`                      | `False` |                                                      |
+| filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
+| filename_suffix\*\* | `str` or `list[str]`        | `None`  | ".lv1"                                               |
+| status\*\*          | `str` or `list[str]`        | `None`  | "created", "uploaded", "processed" or "invalid"      |
 
 \* = only in `metadata()`
 
+\*\* = only in `raw_metadata()`
+
 **Date Handling**
 
 The `date`, `date_from` and `date_to` parameters support:
@@ -84,8 +89,6 @@ Parameters:
 | metadata           | `list[RawMetadata]` or `list[ProductMetadata]` |         |
 | include_pattern    | `str`                                          | `None`  |
 | exclude_pattern    | `str`                                          | `None`  |
-| filename_prefix    | `str`                                          | `None`  |
-| filename_suffix    | `str`                                          | `None`  |
 | include_tag_subset | `set[str]`                                     | `None`  |
 | exclude_tag_subset | `set[str]`                                     | `None`  |
 
@@ -113,7 +116,7 @@ Parameters:
 
 Fetch cloudnet instruments.
 
-### `cloudnet_api_client.download(list[Metadata])` → `list[Path]`
+### `APIClient().download(list[Metadata])` → `list[Path]`
 
 Download files from the fetched metadata.
 

cloudnet_api_client-0.4.0/cloudnet_api_client/__init__.py

@@ -0,0 +1 @@
+from .client import APIClient as APIClient

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/cloudnet_api_client/client.py

@@ -1,7 +1,11 @@
+import asyncio
 import calendar
 import datetime
+import os
 import re
 from dataclasses import fields, is_dataclass
+from os import PathLike
+from pathlib import Path
 from typing import TypeVar, cast
 from urllib.parse import urljoin
 
@@ -12,6 +16,7 @@ from urllib3.util.retry import Retry
 from cloudnet_api_client.containers import (
     PRODUCT_TYPE,
     SITE_TYPE,
+    STATUS,
     Instrument,
     Metadata,
     Product,
@@ -19,11 +24,13 @@ from cloudnet_api_client.containers import (
     RawMetadata,
     Site,
 )
+from cloudnet_api_client.dl import download_files
 
 T = TypeVar("T")
 DateParam = str | datetime.date | None
 DateTimeParam = str | datetime.datetime | datetime.date | None
 QueryParam = str | list[str] | None
+MetadataList = list[ProductMetadata] | list[RawMetadata]
 
 
 class APIClient:
@@ -92,11 +99,18 @@ class APIClient:
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
-
-        files_res = self._get_response("files", params)
+        no_instrument = not instrument_id or instrument_pid
+        if no_instrument and (not product and model_id):
+            files_res = []
+        else:
+            files_res = self._get_response("files", params)
 
         # Add model files if requested
-        if not product or "model" in product:
+        if (
+            (not product and no_instrument)
+            or (product and "model" in product)
+            or (model_id and (not product or "model" in product))
+        ):
             for key in ("showLegacy", "product", "instrument", "instrumentPid"):
                 del params[key]
             params["model"] = model_id
@@ -115,11 +129,17 @@ class APIClient:
         updated_at_to: DateTimeParam = None,
         instrument_id: QueryParam = None,
         instrument_pid: QueryParam = None,
+        filename_prefix: QueryParam = None,
+        filename_suffix: QueryParam = None,
+        status: STATUS | list[STATUS] | None = None,
     ) -> list[RawMetadata]:
         params = {
             "site": site_id,
             "instrument": instrument_id,
             "instrumentPid": instrument_pid,
+            "filenamePrefix": filename_prefix,
+            "filenameSuffix": filename_suffix,
+            "status": status,
         }
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
@@ -127,13 +147,40 @@ class APIClient:
         res = self._get_response("raw-files", params)
         return _build_raw_meta_objects(res)
 
+    def download(
+        self,
+        metadata: MetadataList,
+        output_directory: str | PathLike,
+        concurrency_limit: int = 5,
+        progress: bool | None = None,
+    ) -> list[Path]:
+        return asyncio.run(
+            self.adownload(metadata, output_directory, concurrency_limit, progress)
+        )
+
+    async def adownload(
+        self,
+        metadata: MetadataList,
+        output_directory: str | PathLike,
+        concurrency_limit: int = 5,
+        progress: bool | None = None,
+    ) -> list[Path]:
+        disable_progress = not progress if progress is not None else None
+        output_directory = Path(output_directory).resolve()
+        os.makedirs(output_directory, exist_ok=True)
+        return await download_files(
+            self.base_url,
+            metadata,
+            output_directory,
+            concurrency_limit,
+            disable_progress,
+        )
+
     @staticmethod
     def filter(
         metadata: list[Metadata],
         include_pattern: str | None = None,
         exclude_pattern: str | None = None,
-        filename_prefix: str | None = None,
-        filename_suffix: str | None = None,
         include_tag_subset: set[str] | None = None,
         exclude_tag_subset: set[str] | None = None,
     ) -> list[Metadata]:
@@ -145,10 +192,6 @@ class APIClient:
             metadata = [
                 m for m in metadata if not re.search(exclude_pattern, m.filename, re.I)
             ]
-        if filename_prefix:
-            metadata = [m for m in metadata if m.filename.startswith(filename_prefix)]
-        if filename_suffix:
-            metadata = [m for m in metadata if m.filename.endswith(filename_suffix)]
         if include_tag_subset:
             metadata = [
                 m

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/cloudnet_api_client/containers.py

@@ -4,6 +4,7 @@ from typing import Literal
 
 SITE_TYPE = Literal["cloudnet", "model", "hidden", "campaign"]
 PRODUCT_TYPE = Literal["instrument", "geophysical", "evaluation", "model"]
+STATUS = Literal["created", "uploaded", "processed", "invalid"]
 
 
 @dataclass(frozen=True, slots=True)
@@ -58,7 +59,7 @@ class Metadata:
 
 @dataclass(frozen=True, slots=True)
 class RawMetadata(Metadata):
-    status: Literal["created", "uploaded", "processed", "invalid"]
+    status: STATUS
     instrument: Instrument
     tags: list[str] | None
 

{cloudnet_api_client-0.2.1 → cloudnet_api_client-0.4.0}/cloudnet_api_client/dl.py

@@ -1,7 +1,5 @@
 import asyncio
 import logging
-import os
-from os import PathLike
 from pathlib import Path
 
 import aiohttp
@@ -11,36 +9,10 @@ from tqdm.asyncio import tqdm_asyncio
 from cloudnet_api_client import utils
 from cloudnet_api_client.containers import ProductMetadata, RawMetadata
 
-MetadataList = list[ProductMetadata] | list[RawMetadata]
 
-
-def download(
-    metadata: MetadataList,
-    output_directory: str | PathLike,
-    concurrency_limit: int = 5,
-    progress: bool | None = None,
-) -> list[Path]:
-    return asyncio.run(
-        adownload(metadata, output_directory, concurrency_limit, progress)
-    )
-
-
-async def adownload(
-    metadata: MetadataList,
-    output_directory: str | PathLike,
-    concurrency_limit: int = 5,
-    progress: bool | None = None,
-) -> list[Path]:
-    disable_progress = not progress if progress is not None else None
-    output_directory = Path(output_directory).resolve()
-    os.makedirs(output_directory, exist_ok=True)
-    return await _download_files(
-        metadata, output_directory, concurrency_limit, disable_progress
-    )
-
-
-async def _download_files(
-    metadata: MetadataList,
+async def download_files(
+    base_url: str,
+    metadata: list[ProductMetadata] | list[RawMetadata],
     output_path: Path,
     concurrency_limit: int,
     disable_progress: bool | None,
@@ -50,6 +22,7 @@ async def _download_files(
     async with aiohttp.ClientSession() as session:
         tasks = []
         for meta in metadata:
+            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
             destination = output_path / meta.download_url.split("/")[-1]
             full_paths.append(destination)
             if destination.exists() and _file_checksum_matches(meta, destination):
@@ -57,7 +30,7 @@ async def _download_files(
                 continue
             task = asyncio.create_task(
                 _download_file_with_retries(
-                    session, meta.download_url, destination, semaphore, disable_progress
+                    session, download_url, destination, semaphore, disable_progress
                 )
             )
             tasks.append(task)

cloudnet_api_client-0.4.0/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

cloudnet_api_client-0.2.1/cloudnet_api_client/__init__.py

@@ -1,3 +0,0 @@
-from .client import APIClient as APIClient
-from .dl import adownload as adownload
-from .dl import download as download

cloudnet_api_client-0.2.1/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.2.1"