kleinkram 0.44.0.dev20250409080103__py3-none-any.whl → 0.44.1.dev20250409111712__py3-none-any.whl

kleinkram/api/file_transfer.py

@@ -17,6 +17,7 @@ from uuid import UUID
 import boto3.s3.transfer
 import botocore.config
 import httpx
+from httpx import RemoteProtocolError
 from rich.console import Console
 from tqdm import tqdm
 
@@ -44,6 +45,9 @@ MAX_UPLOAD_RETRIES = 3
 S3_MAX_RETRIES = 60  # same as frontend
 S3_READ_TIMEOUT = 60 * 5  # 5 minutes
 
+RETRY_BACKOFF_BASE = 2  # exponential backoff base
+MAX_RETRIES = 5
+
 
 class UploadCredentials(NamedTuple):
     access_key: str
@@ -264,23 +268,69 @@ def _get_file_download(client: AuthenticatedClient, id: UUID) -> str:
 def _url_download(
     url: str, *, path: Path, size: int, overwrite: bool = False, verbose: bool = False
 ) -> None:
-    if path.exists() and not overwrite:
-        raise FileExistsError(f"file already exists: {path}")
-
-    with httpx.stream("GET", url, timeout=S3_READ_TIMEOUT) as response:
-        response.raise_for_status()
-        with open(path, "wb") as f:
-            with tqdm(
-                total=size,
-                desc=f"downloading {path.name}",
-                unit="B",
-                unit_scale=True,
-                leave=False,
-                disable=not verbose,
-            ) as pbar:
-                for chunk in response.iter_bytes(chunk_size=DOWNLOAD_CHUNK_SIZE):
-                    f.write(chunk)
-                    pbar.update(len(chunk))
+    if path.exists():
+        if overwrite:
+            path.unlink()
+            downloaded = 0
+        else:
+            downloaded = path.stat().st_size
+            if downloaded >= size:
+                raise FileExistsError(f"file already exists and is complete: {path}")
+    else:
+        downloaded = 0
+
+    attempt = 0
+    while downloaded < size:
+        try:
+            headers = {"Range": f"bytes={downloaded}-"}
+            with httpx.stream(
+                "GET", url, headers=headers, timeout=S3_READ_TIMEOUT
+            ) as response:
+                # Accept both 206 Partial Content and 200 OK if starting from 0
+                if not (
+                    response.status_code == 206
+                    or (downloaded == 0 and response.status_code == 200)
+                ):
+                    response.raise_for_status()
+                    raise RuntimeError(
+                        f"Expected 206 Partial Content, got {response.status_code}"
+                    )
+
+                mode = "ab" if downloaded > 0 else "wb"
+                with open(path, mode) as f:
+                    with tqdm(
+                        total=size,
+                        initial=downloaded,
+                        desc=f"downloading {path.name}",
+                        unit="B",
+                        unit_scale=True,
+                        leave=False,
+                        disable=not verbose,
+                    ) as pbar:
+                        for chunk in response.iter_bytes(
+                            chunk_size=DOWNLOAD_CHUNK_SIZE
+                        ):
+                            attempt = 0  # reset attempt counter on successful download of non-empty chunk
+                            if not chunk:
+                                break
+                            f.write(chunk)
+                            downloaded += len(chunk)
+                            pbar.update(len(chunk))
+            break  # download complete
+        except RemoteProtocolError as e:
+            logger.info(f"Error: {e}, retrying...")
+            attempt += 1
+            if attempt > MAX_RETRIES:
+                raise RuntimeError(
+                    f"Download failed after {MAX_RETRIES} retries due to RemoteProtocolError"
+                ) from e
+            if verbose:
+                print(
+                    f"RemoteProtocolError on attempt {attempt}/{MAX_RETRIES}, retrying after backoff..."
+                )
+            sleep(RETRY_BACKOFF_BASE**attempt)
+        except Exception as e:
+            raise RuntimeError(f"Failed to download: {e}") from e
 
 
 class DownloadState(Enum):
@@ -320,7 +370,7 @@ def download_file(
 
             elif verbose:
                 tqdm.write(
-                    styled_string(f"overwriting {path}, hash missmatch", style="yellow")
+                    styled_string(f"overwriting {path}, hash mismatch", style="yellow")
                 )
 
         elif not overwrite and file.size is not None:
@@ -328,9 +378,7 @@ def download_file(
 
         elif verbose:
             tqdm.write(
-                styled_string(
-                    f"overwriting {path}, file size missmatch", style="yellow"
-                )
+                styled_string(f"overwriting {path}, file size mismatch", style="yellow")
             )
 
     # request a download url

{kleinkram-0.44.0.dev20250409080103.dist-info → kleinkram-0.44.1.dev20250409111712.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kleinkram
-Version: 0.44.0.dev20250409080103
+Version: 0.44.1.dev20250409111712
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch

{kleinkram-0.44.0.dev20250409080103.dist-info → kleinkram-0.44.1.dev20250409111712.dist-info}/RECORD

@@ -15,7 +15,7 @@ kleinkram/wrappers.py,sha256=ZScoEov5Q6D2rvaJJ8E-4f58P_NGWrGc9mRPYxSqOC0,13127
 kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kleinkram/api/client.py,sha256=VwuT97_WdbDpcVGwMXB0fRnUoQnUSf7BOP5eXUFokfI,5932
 kleinkram/api/deser.py,sha256=xRpYUFKZ0Luoo7XyAtYblJvprmpjNSZOiFVnFKmOzcM,4819
-kleinkram/api/file_transfer.py,sha256=GYO5CaTQjpSjNg3Nc28kavqfdEy2pgPRkPsnPDSFUPw,18093
+kleinkram/api/file_transfer.py,sha256=9UMEBZP7IV75IHg_tBEnBMQTsT_SIBjG0HDHijAfXco,20094
 kleinkram/api/pagination.py,sha256=P_zPsBKlMWkmAv-YfUNHaGW-XLB_4U8BDMrKyiDFIXk,1370
 kleinkram/api/query.py,sha256=9Exi4hJR7Ml38_zjAcOvSEoIAxZLlpM6QwwzO9fs5Gk,3293
 kleinkram/api/routes.py,sha256=x0IfzQO5RAC3rohDSpdUNlVOWl221gh4e0fbMKfGrQA,12251
@@ -43,8 +43,8 @@ tests/test_printing.py,sha256=Jz1AjqmqBRjp1JLm6H1oVJyvGaMPlahVXdKnd7UDQFc,2231
 tests/test_query.py,sha256=fExmCKXLA7-9j2S2sF_sbvRX_2s6Cp3a7OTcqE25q9g,3864
 tests/test_utils.py,sha256=eUBYrn3xrcgcaxm1X4fqZaX4tRvkbI6rh6BUbNbu9T0,4784
 tests/test_wrappers.py,sha256=TbcTyO2L7fslbzgfDdcVZkencxNQ8cGPZm_iB6c9d6Q,2673
-kleinkram-0.44.0.dev20250409080103.dist-info/METADATA,sha256=slvYitYbGRQnyaZ7BweL9HMUpcMWQUbWgcUdNYeqIJU,2825
-kleinkram-0.44.0.dev20250409080103.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kleinkram-0.44.0.dev20250409080103.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
-kleinkram-0.44.0.dev20250409080103.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
-kleinkram-0.44.0.dev20250409080103.dist-info/RECORD,,
+kleinkram-0.44.1.dev20250409111712.dist-info/METADATA,sha256=2IKtDT39yl-yI4fvJqCsR2oSfLXtAtBp4As7vMQauAE,2825
+kleinkram-0.44.1.dev20250409111712.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kleinkram-0.44.1.dev20250409111712.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
+kleinkram-0.44.1.dev20250409111712.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
+kleinkram-0.44.1.dev20250409111712.dist-info/RECORD,,