kleinkram 0.44.0.dev20250409080103__py3-none-any.whl → 0.44.1__py3-none-any.whl

kleinkram/api/file_transfer.py

@@ -44,6 +44,9 @@ MAX_UPLOAD_RETRIES = 3
 S3_MAX_RETRIES = 60  # same as frontend
 S3_READ_TIMEOUT = 60 * 5  # 5 minutes
 
+RETRY_BACKOFF_BASE = 2  # exponential backoff base
+MAX_RETRIES = 5
+
 
 class UploadCredentials(NamedTuple):
     access_key: str
@@ -264,23 +267,67 @@ def _get_file_download(client: AuthenticatedClient, id: UUID) -> str:
 def _url_download(
     url: str, *, path: Path, size: int, overwrite: bool = False, verbose: bool = False
 ) -> None:
-    if path.exists() and not overwrite:
-        raise FileExistsError(f"file already exists: {path}")
-
-    with httpx.stream("GET", url, timeout=S3_READ_TIMEOUT) as response:
-        response.raise_for_status()
-        with open(path, "wb") as f:
-            with tqdm(
-                total=size,
-                desc=f"downloading {path.name}",
-                unit="B",
-                unit_scale=True,
-                leave=False,
-                disable=not verbose,
-            ) as pbar:
-                for chunk in response.iter_bytes(chunk_size=DOWNLOAD_CHUNK_SIZE):
-                    f.write(chunk)
-                    pbar.update(len(chunk))
+    if path.exists():
+        if overwrite:
+            path.unlink()
+            downloaded = 0
+        else:
+            downloaded = path.stat().st_size
+            if downloaded >= size:
+                raise FileExistsError(f"file already exists and is complete: {path}")
+    else:
+        downloaded = 0
+
+    attempt = 0
+    while downloaded < size:
+        try:
+            headers = {"Range": f"bytes={downloaded}-"}
+            with httpx.stream(
+                "GET", url, headers=headers, timeout=S3_READ_TIMEOUT
+            ) as response:
+                # Accept both 206 Partial Content and 200 OK if starting from 0
+                if not (
+                    response.status_code == 206
+                    or (downloaded == 0 and response.status_code == 200)
+                ):
+                    response.raise_for_status()
+                    raise RuntimeError(
+                        f"Expected 206 Partial Content, got {response.status_code}"
+                    )
+
+                mode = "ab" if downloaded > 0 else "wb"
+                with open(path, mode) as f:
+                    with tqdm(
+                        total=size,
+                        initial=downloaded,
+                        desc=f"downloading {path.name}",
+                        unit="B",
+                        unit_scale=True,
+                        leave=False,
+                        disable=not verbose,
+                    ) as pbar:
+                        for chunk in response.iter_bytes(
+                            chunk_size=DOWNLOAD_CHUNK_SIZE
+                        ):
+                            attempt = 0  # reset attempt counter on successful download of non-empty chunk
+                            if not chunk:
+                                break
+                            f.write(chunk)
+                            downloaded += len(chunk)
+                            pbar.update(len(chunk))
+            break  # download complete
+        except Exception as e:
+            logger.info(f"Error: {e}, retrying...")
+            attempt += 1
+            if attempt > MAX_RETRIES:
+                raise RuntimeError(
+                    f"Download failed after {MAX_RETRIES} retries due to {e}"
+                ) from e
+            if verbose:
+                print(
+                    f"{e} on attempt {attempt}/{MAX_RETRIES}, retrying after backoff..."
+                )
+            sleep(RETRY_BACKOFF_BASE**attempt)
 
 
 class DownloadState(Enum):
@@ -320,7 +367,7 @@ def download_file(
 
             elif verbose:
                 tqdm.write(
-                    styled_string(f"overwriting {path}, hash missmatch", style="yellow")
+                    styled_string(f"overwriting {path}, hash mismatch", style="yellow")
                 )
 
         elif not overwrite and file.size is not None:
@@ -328,9 +375,7 @@ def download_file(
 
         elif verbose:
             tqdm.write(
-                styled_string(
-                    f"overwriting {path}, file size missmatch", style="yellow"
-                )
+                styled_string(f"overwriting {path}, file size mismatch", style="yellow")
             )
 
     # request a download url

kleinkram/printing.py

@@ -85,8 +85,8 @@ def format_bytes(size: int) -> str:
     index = 0
 
     fsize: float = size
-    while fsize >= 1024 and index < len(units) - 1:
-        fsize /= 1024.0
+    while fsize >= 1000 and index < len(units) - 1:
+        fsize /= 1000.0
         index += 1
 
     # Format to 2 decimal places if needed

kleinkram/utils.py

@@ -229,10 +229,10 @@ def format_bytes(size_bytes: int | float, speed: bool = False) -> str:
         return "0 B/s" if speed else "0 B"
 
     units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
-    power = math.floor(math.log(size_bytes, 1024))
+    power = math.floor(math.log(size_bytes, 1000))
     unit_index = min(power, len(units) - 1)
 
-    value = size_bytes / (1024**unit_index)
+    value = size_bytes / (1000**unit_index)
 
     unit_suffix = "/s" if speed else ""
     return f"{value:.2f} {units[unit_index]}{unit_suffix}"

{kleinkram-0.44.0.dev20250409080103.dist-info → kleinkram-0.44.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kleinkram
-Version: 0.44.0.dev20250409080103
+Version: 0.44.1
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch

{kleinkram-0.44.0.dev20250409080103.dist-info → kleinkram-0.44.1.dist-info}/RECORD

@@ -7,15 +7,15 @@ kleinkram/core.py,sha256=COJcUx8tP6EQIyg4OiVprJfQUCMbiq2QGB9xZ2CPUCo,9621
 kleinkram/errors.py,sha256=qa98YvhDbLqX60P8bcMcFmHy4HxgYNlSROXud8Kj-P4,965
 kleinkram/main.py,sha256=BTE0mZN__xd46wBhFi6iBlK9eGGQvJ1LdUMsbnysLi0,172
 kleinkram/models.py,sha256=MkWdGIWuCSnyg45DbdEUhtGIprwPYwKppYumHkRAS18,1870
-kleinkram/printing.py,sha256=q29iYGkBUCmeJNzNacLHRP-8BqF2UhoXirwX7ja7a1k,12212
+kleinkram/printing.py,sha256=gVLQPmAhyO1Fc5kKLBC77KvoEguBaAReTU3h-QQCx58,12212
 kleinkram/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kleinkram/types.py,sha256=nfDjj8TB1Jn5vqO0Xg6qhLOuKom9DDhe62BrngqnVGM,185
-kleinkram/utils.py,sha256=AsKZxEGStn03L2tqqiMVCQCrDyl8HhwOfpa3no4dfYc,6767
+kleinkram/utils.py,sha256=lNtmjKTZj4ANndERqfL3QD8VI2B4ZavdW66lxNDz_IY,6767
 kleinkram/wrappers.py,sha256=ZScoEov5Q6D2rvaJJ8E-4f58P_NGWrGc9mRPYxSqOC0,13127
 kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kleinkram/api/client.py,sha256=VwuT97_WdbDpcVGwMXB0fRnUoQnUSf7BOP5eXUFokfI,5932
 kleinkram/api/deser.py,sha256=xRpYUFKZ0Luoo7XyAtYblJvprmpjNSZOiFVnFKmOzcM,4819
-kleinkram/api/file_transfer.py,sha256=GYO5CaTQjpSjNg3Nc28kavqfdEy2pgPRkPsnPDSFUPw,18093
+kleinkram/api/file_transfer.py,sha256=eHJlK1FY0hHBmFHAves_p49hi4Mmpdjy1NNLeyLvK0w,19917
 kleinkram/api/pagination.py,sha256=P_zPsBKlMWkmAv-YfUNHaGW-XLB_4U8BDMrKyiDFIXk,1370
 kleinkram/api/query.py,sha256=9Exi4hJR7Ml38_zjAcOvSEoIAxZLlpM6QwwzO9fs5Gk,3293
 kleinkram/api/routes.py,sha256=x0IfzQO5RAC3rohDSpdUNlVOWl221gh4e0fbMKfGrQA,12251
@@ -39,12 +39,12 @@ tests/test_core.py,sha256=tVvmC4yBkE00VspoowhfdNAUiBsmkIJQodaC3jjUqqM,5623
 tests/test_end_to_end.py,sha256=0W5pUES5hek-pXq4NZtpPZqKTORkGCRsDv5_D3rDMjY,3372
 tests/test_error_handling.py,sha256=qPSMKF1qsAHyUME0-krxbIrk38iGKkhAyAah-KwN4NE,1300
 tests/test_fixtures.py,sha256=UlPmGbEsGvrDPsaStGMRjNvrVPGjCqOB0RMfLJq2VRA,1071
-tests/test_printing.py,sha256=Jz1AjqmqBRjp1JLm6H1oVJyvGaMPlahVXdKnd7UDQFc,2231
+tests/test_printing.py,sha256=kPzpIQOtQJ9yQ32mM8cMGDVOGsbrZZLQhfsXN1Pe68Q,2231
 tests/test_query.py,sha256=fExmCKXLA7-9j2S2sF_sbvRX_2s6Cp3a7OTcqE25q9g,3864
 tests/test_utils.py,sha256=eUBYrn3xrcgcaxm1X4fqZaX4tRvkbI6rh6BUbNbu9T0,4784
 tests/test_wrappers.py,sha256=TbcTyO2L7fslbzgfDdcVZkencxNQ8cGPZm_iB6c9d6Q,2673
-kleinkram-0.44.0.dev20250409080103.dist-info/METADATA,sha256=slvYitYbGRQnyaZ7BweL9HMUpcMWQUbWgcUdNYeqIJU,2825
-kleinkram-0.44.0.dev20250409080103.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kleinkram-0.44.0.dev20250409080103.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
-kleinkram-0.44.0.dev20250409080103.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
-kleinkram-0.44.0.dev20250409080103.dist-info/RECORD,,
+kleinkram-0.44.1.dist-info/METADATA,sha256=yX6R8JZh8xG0FcfjE8fUaKOAfPoAxbg4TV_UN6k73ac,2807
+kleinkram-0.44.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kleinkram-0.44.1.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
+kleinkram-0.44.1.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
+kleinkram-0.44.1.dist-info/RECORD,,

tests/test_printing.py

@@ -15,14 +15,14 @@ from kleinkram.printing import parse_metadata_value
 def test_format_bytes():
     assert format_bytes(0) == "0 B"
     assert format_bytes(1) == "1 B"
-    assert format_bytes(1000) == "1000 B"
-    assert format_bytes(1024) == "1.00 KB"
-    assert format_bytes(1025) == "1.00 KB"
-    assert format_bytes(2048) == "2.00 KB"
-    assert format_bytes(2**20) == "1.00 MB"
-    assert format_bytes(2**30) == "1.00 GB"
-    assert format_bytes(2**40) == "1.00 TB"
-    assert format_bytes(2**50) == "1.00 PB"
+    assert format_bytes(999) == "999 B"
+    assert format_bytes(1000) == "1.00 KB"
+    assert format_bytes(1001) == "1.00 KB"
+    assert format_bytes(2000) == "2.00 KB"
+    assert format_bytes(10**6) == "1.00 MB"
+    assert format_bytes(10**9) == "1.00 GB"
+    assert format_bytes(10**12) == "1.00 TB"
+    assert format_bytes(10**15) == "1.00 PB"
 
 
 def test_add_placeholder_row():