cloudnet-api-client 0.12.6__tar.gz → 0.12.8__tar.gz

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/.github/dataportal.env

@@ -8,8 +8,8 @@ TYPEORM_PORT=5432
 TYPEORM_SYNCHRONIZE=false
 TYPEORM_MIGRATIONS_RUN=true
 TYPEORM_LOGGING=false
-TYPEORM_ENTITIES=build/entity/*.js
-TYPEORM_MIGRATIONS=build/migration/*.js
+TYPEORM_ENTITIES=build/app/src/entity/*.js
+TYPEORM_MIGRATIONS=build/app/src/migration/*.js
 DP_SS_URL=http://storage-service:5900
 DP_SS_USER=test
 DP_SS_PASSWORD=test
@@ -24,6 +24,6 @@ DATACITE_API_TIMEOUT_MS=2000
 DATACITE_DOI_SERVER=http://handle.datacite.test
 DATACITE_DOI_PREFIX=XXX
 LABELLING_URL=http://localhost:5803
-HANDLE_API_URL=http://localhost:5804
+INSTRUMENTDB_URL=http://localhost:5805
 DVAS_URL=https://dvas.test
 DC_URL=https://dc.test

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/.github/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       [
         "sh",
         "-c",
-        "node build/fixtures.js /backend-fixtures/backend/fixtures TRUNCATE && node build/fixtures.js /dataportal-fixtures APPEND && npm run start",
+        "node build/app/src/fixtures.js /backend-fixtures/backend/fixtures TRUNCATE && node build/app/src/fixtures.js /dataportal-fixtures APPEND && npm run start",
       ]
   db:
     image: "postgres:16"

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/.github/workflows/test.yml

@@ -62,3 +62,7 @@ jobs:
       - name: Shutdown backend
         if: always()
         run: docker compose -f .github/docker-compose.yml down
+
+      - name: Print logs
+        if: always()
+        run: docker compose -f .github/docker-compose.yml logs

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.12.8 – 2026-03-24
+
+- Support downloading of single metadata objects
+- Hide total progress bar when downloading a single file
+
+## 0.12.7 – 2025-12-18
+
+- Adjust progress bars
+- Write partial file
+
 ## 0.12.6 – 2025-09-17
 
 - Allow other iterables besides `list` as argument

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.12.6
+Version: 0.12.8
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -51,7 +51,7 @@ sites = client.sites()
 site = client.site("hyytiala")
 
 products = client.products()
-product = client.products("classification")
+product = client.product("classification")
 
 models = client.models()
 model = client.model("ecmwf-open")
@@ -239,13 +239,13 @@ Download files from the fetched metadata.
 
 Parameters:
 
-| name              | type                                           | default           |
-| ----------------- | ---------------------------------------------- | ----------------- |
-| metadata          | `list[RawMetadata]` or `list[ProductMetadata]` |                   |
-| output_directory  | `PathLike` or `str`                            | Current directory |
-| concurrency_limit | `int`                                          | 5                 |
-| progress          | `bool` or `None`                               | `None`            |
-| validate_checksum | `bool`                                         | `False`           |
+| name              | type                                            | default           |
+| ----------------- | ----------------------------------------------- | ----------------- |
+| metadata          | `RawMetadata`, `ProductMetadata` or `list[...]` |                   |
+| output_directory  | `PathLike` or `str`                             | Current directory |
+| concurrency_limit | `int`                                           | 5                 |
+| progress          | `bool` or `None`                                | `None`            |
+| validate_checksum | `bool`                                          | `False`           |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/README.md

@@ -22,7 +22,7 @@ sites = client.sites()
 site = client.site("hyytiala")
 
 products = client.products()
-product = client.products("classification")
+product = client.product("classification")
 
 models = client.models()
 model = client.model("ecmwf-open")
@@ -210,13 +210,13 @@ Download files from the fetched metadata.
 
 Parameters:
 
-| name              | type                                           | default           |
-| ----------------- | ---------------------------------------------- | ----------------- |
-| metadata          | `list[RawMetadata]` or `list[ProductMetadata]` |                   |
-| output_directory  | `PathLike` or `str`                            | Current directory |
-| concurrency_limit | `int`                                          | 5                 |
-| progress          | `bool` or `None`                               | `None`            |
-| validate_checksum | `bool`                                         | `False`           |
+| name              | type                                            | default           |
+| ----------------- | ----------------------------------------------- | ----------------- |
+| metadata          | `RawMetadata`, `ProductMetadata` or `list[...]` |                   |
+| output_directory  | `PathLike` or `str`                             | Current directory |
+| concurrency_limit | `int`                                           | 5                 |
+| progress          | `bool` or `None`                                | `None`            |
+| validate_checksum | `bool`                                          | `False`           |
 
 There's also an asynchronous version of this function:
 `cloudnet_api_client.adownload`. It's useful for usage inside Jupyter notebook.

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/cloudnet_api_client/client.py

@@ -322,7 +322,7 @@ class APIClient:
 
     def download(
         self,
-        metadata: MetadataList,
+        metadata: MetadataList | TMetadata,
         output_directory: str | PathLike = ".",
         concurrency_limit: int = 5,
         progress: bool | None = None,
@@ -340,7 +340,7 @@ class APIClient:
 
     async def adownload(
         self,
-        metadata: MetadataList,
+        metadata: MetadataList | TMetadata,
         output_directory: str | PathLike = ".",
         concurrency_limit: int = 5,
         progress: bool | None = None,

cloudnet_api_client-0.12.8/cloudnet_api_client/dl.py

@@ -0,0 +1,175 @@
+import asyncio
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+import aiohttp
+from tqdm import tqdm
+
+from cloudnet_api_client import utils
+from cloudnet_api_client.containers import (
+    Metadata,
+    ProductMetadata,
+)
+
+
+class BarConfig:
+    def __init__(
+        self,
+        disable: bool | None,
+        max_workers: int,
+        total_bytes: int,
+        n_files: int,
+    ) -> None:
+        self.disable = disable
+        self.single_file = n_files <= 1
+        self.position_queue = self._init_position_queue(max_workers)
+        self.total_amount = tqdm(
+            total=total_bytes,
+            desc="Progress",
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+            disable=self.disable if not self.single_file else True,
+            position=0,
+            leave=False,
+            colour="green",
+        )
+        self.lock = asyncio.Lock()
+
+    def _init_position_queue(self, max_workers: int) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        start = 0 if self.single_file else 1
+        for i in range(start, start + max_workers):
+            queue.put_nowait(i)
+        return queue
+
+
+@dataclass
+class DlParams:
+    url: str
+    destination: Path
+    session: aiohttp.ClientSession
+    semaphore: asyncio.Semaphore
+    bar_config: BarConfig
+    disable: bool | None
+
+
+async def download_files(
+    base_url: str,
+    metadata: Iterable[Metadata] | Metadata,
+    output_path: Path,
+    concurrency_limit: int,
+    disable_progress: bool | None,
+    validate_checksum: bool = False,
+) -> list[Path]:
+    metas = list(metadata) if isinstance(metadata, Iterable) else [metadata]
+    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
+    semaphore = asyncio.Semaphore(concurrency_limit)
+    total_bytes = sum(meta.size for meta in metas)
+    bar_config = BarConfig(disable_progress, concurrency_limit, total_bytes, len(metas))
+    full_paths = []
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for meta in metas:
+            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
+            destination = output_path / meta.download_url.split("/")[-1]
+            full_paths.append(destination)
+            if destination.exists() and file_exists(meta, destination):
+                logging.debug(f"Already downloaded: {destination}")
+                continue
+            dl_params = DlParams(
+                url=download_url,
+                destination=destination,
+                session=session,
+                semaphore=semaphore,
+                bar_config=bar_config,
+                disable=disable_progress,
+            )
+            task = asyncio.create_task(_download_file_with_retries(dl_params))
+            tasks.append(task)
+        if disable_progress is True:
+            print(f"Downloading {len(metas)} files...", end="", flush=True)
+        await asyncio.gather(*tasks)
+        bar_config.total_amount.close()
+        bar_config.total_amount.clear()
+    if disable_progress is True:
+        print(" done.", flush=True)
+    return full_paths
+
+
+async def _download_file_with_retries(
+    params: DlParams,
+    max_retries: int = 3,
+) -> None:
+    """Attempt to download a file, retrying up to max_retries times if needed."""
+    position = await params.bar_config.position_queue.get()
+    try:
+        for attempt in range(1, max_retries + 1):
+            try:
+                await _download_file(params, position)
+                return
+            except aiohttp.ClientError as e:
+                logging.warning(f"Attempt {attempt} failed for {params.url}: {e}")
+                if attempt == max_retries:
+                    logging.error(
+                        f"Giving up on {params.url} after {max_retries} attempts."
+                    )
+                    raise e
+                else:
+                    # Exponential backoff before retrying
+                    await asyncio.sleep(2**attempt)
+    finally:
+        params.bar_config.position_queue.put_nowait(position)
+    raise RuntimeError("Unreachable code reached.")
+
+
+async def _download_file(
+    params: DlParams,
+    position: int,
+) -> None:
+    tmp_path = params.destination.with_suffix(f"{params.destination.suffix}.part")
+    async with params.semaphore, params.session.get(params.url) as response:
+        response.raise_for_status()
+        bar = tqdm(
+            desc=params.destination.name,
+            total=response.content_length,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+            disable=params.bar_config.disable,
+            position=position,
+            leave=False,
+            colour="cyan",
+        )
+        try:
+            tmp_path.parent.mkdir(parents=True, exist_ok=True)
+            with tmp_path.open("wb") as f:
+                async for chunk in response.content.iter_chunked(8192):
+                    f.write(chunk)
+                    bar.update(len(chunk))
+                    params.bar_config.total_amount.update(len(chunk))
+            tmp_path.replace(params.destination)
+        except Exception:
+            try:
+                if tmp_path.exists():
+                    tmp_path.unlink()
+            except OSError:
+                pass
+            raise
+        finally:
+            bar.close()
+            bar.clear()
+
+
+def _checksum_matches(meta: Metadata, destination: Path) -> bool:
+    fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
+    return fun(destination) == meta.checksum
+
+
+def _size_and_name_matches(meta: Metadata, destination: Path) -> bool:
+    return (
+        destination.stat().st_size == meta.size
+        and destination.name == meta.download_url.split("/")[-1]
+    )

cloudnet_api_client-0.12.8/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.12.8"

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.8}/tests/test_client.py

@@ -382,7 +382,7 @@ class TestRawFiles:
 
     def test_filter_by_filename_suffix(self, client: APIClient):
         meta = client.raw_files(filename_suffix="000.nc")
-        assert len(meta) == 2
+        assert len(meta) == 3
 
     def test_filter_by_instrument_id(self, client: APIClient):
         meta = client.raw_files(instrument_id="weather-station")
@@ -462,6 +462,19 @@ class TestDownloadingFunctionality:
         assert paths1 == paths2
         assert paths2[0].stat().st_size == original_size
 
+    def test_downloading_single_metadata(self, client: APIClient, tmp_path: Path):
+        uuid = "ab872770-9136-4e61-8958-31e62abdfb1b"
+        meta = client.file(uuid)
+        paths = client.download(meta, output_directory=tmp_path, progress=False)
+        assert len(paths) == 1
+        assert paths[0].exists()
+
+    def test_downloading_single_metadata_II(self, client: APIClient, tmp_path: Path):
+        meta = client.raw_files(date_from="2025-08-01")
+        assert len(meta) == 3
+        paths = client.download(meta[0], output_directory=tmp_path, progress=False)
+        assert len(paths) == 1
+
     async def test_async_download(self, client: APIClient, tmp_path: Path):
         meta = client.raw_files(date_from="2025-08-01")
         assert len(meta) == 3
@@ -487,7 +500,7 @@ class TestFilterCombinations:
         assert len(meta) == 0
 
     def test_partial_filename_matches(self, client: APIClient):
-        meta = client.raw_files(filename_prefix="2025", filename_suffix=".nc")
+        meta = client.raw_files(filename_prefix="202508", filename_suffix=".nc")
         assert len(meta) == 2
 
 

cloudnet_api_client-0.12.6/cloudnet_api_client/dl.py

@@ -1,108 +0,0 @@
-import asyncio
-import logging
-from collections.abc import Iterable
-from pathlib import Path
-
-import aiohttp
-from tqdm import tqdm
-from tqdm.asyncio import tqdm_asyncio
-
-from cloudnet_api_client import utils
-from cloudnet_api_client.containers import Metadata, ProductMetadata
-
-
-async def download_files(
-    base_url: str,
-    metadata: Iterable[Metadata],
-    output_path: Path,
-    concurrency_limit: int,
-    disable_progress: bool | None,
-    validate_checksum: bool = False,
-) -> list[Path]:
-    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
-    semaphore = asyncio.Semaphore(concurrency_limit)
-    full_paths = []
-    async with aiohttp.ClientSession() as session:
-        tasks = []
-        for meta in metadata:
-            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
-            destination = output_path / meta.download_url.split("/")[-1]
-            full_paths.append(destination)
-            if destination.exists() and file_exists(meta, destination):
-                logging.debug(f"Already downloaded: {destination}")
-                continue
-            task = asyncio.create_task(
-                _download_file_with_retries(
-                    session, download_url, destination, semaphore, disable_progress
-                )
-            )
-            tasks.append(task)
-        await tqdm_asyncio.gather(
-            *tasks, desc="Completed files", disable=disable_progress
-        )
-    return full_paths
-
-
-async def _download_file_with_retries(
-    session: aiohttp.ClientSession,
-    url: str,
-    destination: Path,
-    semaphore: asyncio.Semaphore,
-    disable_progress: bool | None,
-    max_retries: int = 3,
-) -> None:
-    """Attempt to download a file, retrying up to max_retries times if needed."""
-    for attempt in range(1, max_retries + 1):
-        try:
-            await _download_file(session, url, destination, semaphore, disable_progress)
-            return
-        except aiohttp.ClientError as e:
-            logging.warning(f"Attempt {attempt} failed for {url}: {e}")
-            if attempt == max_retries:
-                logging.error(f"Giving up on {url} after {max_retries} attempts.")
-                raise e
-            else:
-                # Exponential backoff before retrying
-                await asyncio.sleep(2**attempt)
-
-
-async def _download_file(
-    session: aiohttp.ClientSession,
-    url: str,
-    destination: Path,
-    semaphore: asyncio.Semaphore,
-    disable_progress: bool | None,
-) -> None:
-    async with semaphore:
-        async with session.get(url) as response:
-            response.raise_for_status()
-            with (
-                destination.open("wb") as file_out,
-                tqdm(
-                    desc=destination.name,
-                    total=response.content_length,
-                    unit="iB",
-                    unit_scale=True,
-                    unit_divisor=1024,
-                    disable=disable_progress,
-                ) as bar,
-            ):
-                while True:
-                    chunk = await response.content.read(8192)
-                    if not chunk:
-                        break
-                    file_out.write(chunk)
-                    bar.update(len(chunk))
-        logging.debug(f"Downloaded: {destination}")
-
-
-def _checksum_matches(meta: Metadata, destination: Path) -> bool:
-    fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
-    return fun(destination) == meta.checksum
-
-
-def _size_and_name_matches(meta: Metadata, destination: Path) -> bool:
-    return (
-        destination.stat().st_size == meta.size
-        and destination.name == meta.download_url.split("/")[-1]
-    )

cloudnet_api_client-0.12.6/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.12.6"