cloudnet-api-client 0.12.6__tar.gz → 0.12.7__tar.gz

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.7}/.github/dataportal.env

@@ -24,6 +24,6 @@ DATACITE_API_TIMEOUT_MS=2000
 DATACITE_DOI_SERVER=http://handle.datacite.test
 DATACITE_DOI_PREFIX=XXX
 LABELLING_URL=http://localhost:5803
-HANDLE_API_URL=http://localhost:5804
+INSTRUMENTDB_URL=http://localhost:5805
 DVAS_URL=https://dvas.test
 DC_URL=https://dc.test

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.7}/CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.12.7 – 2025-12-18
+
+- Adjust progress bars
+- Write partial file
+
 ## 0.12.6 – 2025-09-17
 
 - Allow other iterables besides `list` as argument

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.12.6
+Version: 0.12.7
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE

cloudnet_api_client-0.12.7/cloudnet_api_client/dl.py

@@ -0,0 +1,166 @@
+import asyncio
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+import aiohttp
+from tqdm import tqdm
+
+from cloudnet_api_client import utils
+from cloudnet_api_client.containers import Metadata, ProductMetadata
+
+
+class BarConfig:
+    def __init__(
+        self, disable: bool | None, max_workers: int, total_bytes: int
+    ) -> None:
+        self.disable = disable
+        self.position_queue = self._init_position_queue(max_workers)
+        self.total_amount = tqdm(
+            total=total_bytes,
+            desc="Progress",
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+            disable=self.disable,
+            position=0,
+            leave=False,
+            colour="green",
+        )
+        self.lock = asyncio.Lock()
+
+    def _init_position_queue(self, max_workers: int) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        for i in range(1, max_workers + 1):
+            queue.put_nowait(i)
+        return queue
+
+
+@dataclass
+class DlParams:
+    url: str
+    destination: Path
+    session: aiohttp.ClientSession
+    semaphore: asyncio.Semaphore
+    bar_config: BarConfig
+    disable: bool | None
+
+
+async def download_files(
+    base_url: str,
+    metadata: Iterable[Metadata],
+    output_path: Path,
+    concurrency_limit: int,
+    disable_progress: bool | None,
+    validate_checksum: bool = False,
+) -> list[Path]:
+    metas = list(metadata)
+    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
+    semaphore = asyncio.Semaphore(concurrency_limit)
+    total_bytes = sum(meta.size for meta in metas)
+    bar_config = BarConfig(disable_progress, concurrency_limit, total_bytes)
+    full_paths = []
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for meta in metas:
+            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
+            destination = output_path / meta.download_url.split("/")[-1]
+            full_paths.append(destination)
+            if destination.exists() and file_exists(meta, destination):
+                logging.debug(f"Already downloaded: {destination}")
+                continue
+            dl_params = DlParams(
+                url=download_url,
+                destination=destination,
+                session=session,
+                semaphore=semaphore,
+                bar_config=bar_config,
+                disable=disable_progress,
+            )
+            task = asyncio.create_task(_download_file_with_retries(dl_params))
+            tasks.append(task)
+        if disable_progress is True:
+            print(f"Downloading {len(metas)} files...", end="", flush=True)
+        await asyncio.gather(*tasks)
+        bar_config.total_amount.close()
+        bar_config.total_amount.clear()
+    if disable_progress is True:
+        print(" done.", flush=True)
+    return full_paths
+
+
+async def _download_file_with_retries(
+    params: DlParams,
+    max_retries: int = 3,
+) -> None:
+    """Attempt to download a file, retrying up to max_retries times if needed."""
+    position = await params.bar_config.position_queue.get()
+    try:
+        for attempt in range(1, max_retries + 1):
+            try:
+                await _download_file(params, position)
+                return
+            except aiohttp.ClientError as e:
+                logging.warning(f"Attempt {attempt} failed for {params.url}: {e}")
+                if attempt == max_retries:
+                    logging.error(
+                        f"Giving up on {params.url} after {max_retries} attempts."
+                    )
+                    raise e
+                else:
+                    # Exponential backoff before retrying
+                    await asyncio.sleep(2**attempt)
+    finally:
+        params.bar_config.position_queue.put_nowait(position)
+    raise RuntimeError("Unreachable code reached.")
+
+
+async def _download_file(
+    params: DlParams,
+    position: int,
+) -> None:
+    tmp_path = params.destination.with_suffix(f"{params.destination.suffix}.part")
+    async with params.semaphore, params.session.get(params.url) as response:
+        response.raise_for_status()
+        bar = tqdm(
+            desc=params.destination.name,
+            total=response.content_length,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+            disable=params.bar_config.disable,
+            position=position,
+            leave=False,
+            colour="cyan",
+        )
+        try:
+            tmp_path.parent.mkdir(parents=True, exist_ok=True)
+            with tmp_path.open("wb") as f:
+                async for chunk in response.content.iter_chunked(8192):
+                    f.write(chunk)
+                    bar.update(len(chunk))
+                    params.bar_config.total_amount.update(len(chunk))
+            tmp_path.replace(params.destination)
+        except Exception:
+            try:
+                if tmp_path.exists():
+                    tmp_path.unlink()
+            except OSError:
+                pass
+            raise
+        finally:
+            bar.close()
+            bar.clear()
+
+
+def _checksum_matches(meta: Metadata, destination: Path) -> bool:
+    fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
+    return fun(destination) == meta.checksum
+
+
+def _size_and_name_matches(meta: Metadata, destination: Path) -> bool:
+    return (
+        destination.stat().st_size == meta.size
+        and destination.name == meta.download_url.split("/")[-1]
+    )

cloudnet_api_client-0.12.7/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.12.7"

{cloudnet_api_client-0.12.6 → cloudnet_api_client-0.12.7}/tests/test_client.py

@@ -382,7 +382,7 @@ class TestRawFiles:
 
     def test_filter_by_filename_suffix(self, client: APIClient):
         meta = client.raw_files(filename_suffix="000.nc")
-        assert len(meta) == 2
+        assert len(meta) == 3
 
     def test_filter_by_instrument_id(self, client: APIClient):
         meta = client.raw_files(instrument_id="weather-station")
@@ -487,7 +487,7 @@ class TestFilterCombinations:
         assert len(meta) == 0
 
     def test_partial_filename_matches(self, client: APIClient):
-        meta = client.raw_files(filename_prefix="2025", filename_suffix=".nc")
+        meta = client.raw_files(filename_prefix="202508", filename_suffix=".nc")
         assert len(meta) == 2
 
 

cloudnet_api_client-0.12.6/cloudnet_api_client/dl.py

@@ -1,108 +0,0 @@
-import asyncio
-import logging
-from collections.abc import Iterable
-from pathlib import Path
-
-import aiohttp
-from tqdm import tqdm
-from tqdm.asyncio import tqdm_asyncio
-
-from cloudnet_api_client import utils
-from cloudnet_api_client.containers import Metadata, ProductMetadata
-
-
-async def download_files(
-    base_url: str,
-    metadata: Iterable[Metadata],
-    output_path: Path,
-    concurrency_limit: int,
-    disable_progress: bool | None,
-    validate_checksum: bool = False,
-) -> list[Path]:
-    file_exists = _checksum_matches if validate_checksum else _size_and_name_matches
-    semaphore = asyncio.Semaphore(concurrency_limit)
-    full_paths = []
-    async with aiohttp.ClientSession() as session:
-        tasks = []
-        for meta in metadata:
-            download_url = f"{base_url}{meta.download_url.split('/api/')[-1]}"
-            destination = output_path / meta.download_url.split("/")[-1]
-            full_paths.append(destination)
-            if destination.exists() and file_exists(meta, destination):
-                logging.debug(f"Already downloaded: {destination}")
-                continue
-            task = asyncio.create_task(
-                _download_file_with_retries(
-                    session, download_url, destination, semaphore, disable_progress
-                )
-            )
-            tasks.append(task)
-        await tqdm_asyncio.gather(
-            *tasks, desc="Completed files", disable=disable_progress
-        )
-    return full_paths
-
-
-async def _download_file_with_retries(
-    session: aiohttp.ClientSession,
-    url: str,
-    destination: Path,
-    semaphore: asyncio.Semaphore,
-    disable_progress: bool | None,
-    max_retries: int = 3,
-) -> None:
-    """Attempt to download a file, retrying up to max_retries times if needed."""
-    for attempt in range(1, max_retries + 1):
-        try:
-            await _download_file(session, url, destination, semaphore, disable_progress)
-            return
-        except aiohttp.ClientError as e:
-            logging.warning(f"Attempt {attempt} failed for {url}: {e}")
-            if attempt == max_retries:
-                logging.error(f"Giving up on {url} after {max_retries} attempts.")
-                raise e
-            else:
-                # Exponential backoff before retrying
-                await asyncio.sleep(2**attempt)
-
-
-async def _download_file(
-    session: aiohttp.ClientSession,
-    url: str,
-    destination: Path,
-    semaphore: asyncio.Semaphore,
-    disable_progress: bool | None,
-) -> None:
-    async with semaphore:
-        async with session.get(url) as response:
-            response.raise_for_status()
-            with (
-                destination.open("wb") as file_out,
-                tqdm(
-                    desc=destination.name,
-                    total=response.content_length,
-                    unit="iB",
-                    unit_scale=True,
-                    unit_divisor=1024,
-                    disable=disable_progress,
-                ) as bar,
-            ):
-                while True:
-                    chunk = await response.content.read(8192)
-                    if not chunk:
-                        break
-                    file_out.write(chunk)
-                    bar.update(len(chunk))
-        logging.debug(f"Downloaded: {destination}")
-
-
-def _checksum_matches(meta: Metadata, destination: Path) -> bool:
-    fun = utils.sha256sum if isinstance(meta, ProductMetadata) else utils.md5sum
-    return fun(destination) == meta.checksum
-
-
-def _size_and_name_matches(meta: Metadata, destination: Path) -> bool:
-    return (
-        destination.stat().st_size == meta.size
-        and destination.name == meta.download_url.split("/")[-1]
-    )

cloudnet_api_client-0.12.6/cloudnet_api_client/version.py

@@ -1 +0,0 @@
-__version__ = "0.12.6"