kleinkram 0.43.6.dev20250401131403__py3-none-any.whl → 0.44.0__py3-none-any.whl

kleinkram/api/file_transfer.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import signal
 import logging
 import sys
 from concurrent.futures import Future
@@ -18,6 +17,9 @@ from uuid import UUID
 import boto3.s3.transfer
 import botocore.config
 import httpx
+from rich.console import Console
+from tqdm import tqdm
+
 from kleinkram.api.client import AuthenticatedClient
 from kleinkram.config import get_config
 from kleinkram.errors import AccessDenied
@@ -28,8 +30,6 @@ from kleinkram.utils import format_bytes
 from kleinkram.utils import format_error
 from kleinkram.utils import format_traceback
 from kleinkram.utils import styled_string
-from rich.console import Console
-from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
@@ -267,7 +267,7 @@ def _url_download(
     if path.exists() and not overwrite:
         raise FileExistsError(f"file already exists: {path}")
 
-    with httpx.stream("GET", url) as response:
+    with httpx.stream("GET", url, timeout=S3_READ_TIMEOUT) as response:
         response.raise_for_status()
         with open(path, "wb") as f:
             with tqdm(
@@ -289,6 +289,7 @@ class DownloadState(Enum):
     DOWNLOADED_INVALID_HASH = 3
     SKIPPED_INVALID_HASH = 4
     SKIPPED_INVALID_REMOTE_STATE = 5
+    SKIPPED_FILE_SIZE_MISMATCH = 6
 
 
 def download_file(
@@ -307,17 +308,27 @@ def download_file(
         return DownloadState.SKIPPED_INVALID_REMOTE_STATE, 0
 
     if path.exists():
-        local_hash = b64_md5(path)
-        if local_hash != file.hash and not overwrite and file.hash is not None:
-            return DownloadState.SKIPPED_INVALID_HASH
 
-        elif local_hash == file.hash:
-            return DownloadState.SKIPPED_OK
+        # compare file size
+        if file.size == path.stat().st_size:
+            local_hash = b64_md5(path)
+            if local_hash != file.hash and not overwrite and file.hash is not None:
+                return DownloadState.SKIPPED_INVALID_HASH, 0
 
-        # this has to be here
-        if verbose:
+            elif local_hash == file.hash:
+                return DownloadState.SKIPPED_OK, 0
+
+            elif verbose:
+                tqdm.write(
+                    styled_string(f"overwriting {path}, hash mismatch", style="yellow")
+                )
+
+        elif not overwrite and file.size is not None:
+            return DownloadState.SKIPPED_FILE_SIZE_MISMATCH, 0
+
+        elif verbose:
             tqdm.write(
-                styled_string(f"overwriting {path}, hash missmatch", style="yellow")
+                styled_string(f"overwriting {path}, file size mismatch", style="yellow")
             )
 
     # request a download url
@@ -400,6 +411,7 @@ DOWNLOAD_STATE_COLOR = {
     DownloadState.SKIPPED_OK: "green",
     DownloadState.DOWNLOADED_INVALID_HASH: "red",
     DownloadState.SKIPPED_INVALID_HASH: "yellow",
+    DownloadState.SKIPPED_FILE_SIZE_MISMATCH: "yellow",
     DownloadState.SKIPPED_INVALID_REMOTE_STATE: "purple",
 }
 
@@ -432,6 +444,8 @@ def _download_handler(
         msg = f"skipped {path} already downloaded (hash ok)"
     elif state == DownloadState.SKIPPED_INVALID_HASH:
         msg = f"skipped {path}, exists with hash mismatch (use --overwrite?)"
+    elif state == DownloadState.SKIPPED_FILE_SIZE_MISMATCH:
+        msg = f"skipped {path}, exists with file size mismatch (use --overwrite?)"
     elif state == DownloadState.SKIPPED_INVALID_REMOTE_STATE:
         msg = f"skipped {path}, remote file has invalid state ({file.state.value})"
     else:

kleinkram/cli/_download.py

@@ -43,7 +43,8 @@ def download(
         False, help="save files in nested directories, project-name/mission-name"
     ),
     overwrite: bool = typer.Option(
-        False, help="overwrite files if they already exist and don't match the filehash"
+        False,
+        help="overwrite files if they already exist and don't match the file size or file hash",
     ),
 ) -> None:
     # create destionation directory

kleinkram/cli/_verify.py

@@ -32,7 +32,15 @@ def verify(
         None, "--project", "-p", help="project id or name"
     ),
     mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
-    skip_hash: bool = typer.Option(False, help="skip hash check"),
+    skip_hash: bool = typer.Option(None, help="skip hash check"),
+    check_file_hash: bool = typer.Option(
+        True,
+        help="check file hash. If True, file names and file hashes are checked.",
+    ),
+    check_file_size: bool = typer.Option(
+        True,
+        help="check file size. If True, file names and file sizes are checked.",
+    ),
 ) -> None:
     # get all filepaths
     file_paths = [Path(file) for file in files]
@@ -51,6 +59,8 @@ def verify(
         query=mission_query,
         file_paths=file_paths,
         skip_hash=skip_hash,
+        check_file_hash=check_file_hash,
+        check_file_size=check_file_size,
         verbose=verbose,
     )
     print_file_verification_status(file_status, pprint=verbose)

kleinkram/core.py

@@ -133,9 +133,20 @@ def verify(
     client: AuthenticatedClient,
     query: MissionQuery,
     file_paths: Sequence[Path],
-    skip_hash: bool = False,
+    skip_hash: Optional[bool] = None,
+    check_file_hash: bool = True,
+    check_file_size: bool = False,
     verbose: bool = False,
 ) -> Dict[Path, FileVerificationStatus]:
+
+    # add deprecated warning for skip_hash
+    if skip_hash is not None:
+        print(
+            "Warning: --skip-hash is deprecated and will be removed in a future version. "
+            "Use --check-file-hash=False instead.",
+        )
+        check_file_hash = not skip_hash
+
     # check that file paths are for valid files and have valid suffixes
     check_file_paths(file_paths)
 
@@ -167,14 +178,30 @@ def verify(
         if remote_file.state == FileState.UPLOADING:
             file_status[file] = FileVerificationStatus.UPLOADING
         elif remote_file.state == FileState.OK:
-            if remote_file.hash is None:
-                file_status[file] = FileVerificationStatus.COMPUTING_HASH
-            elif skip_hash or remote_file.hash == b64_md5(file):
-                file_status[file] = FileVerificationStatus.UPLAODED
-            else:
-                file_status[file] = FileVerificationStatus.MISMATCHED_HASH
+
+            # default case, will be overwritten if we find a mismatch
+            file_status[file] = FileVerificationStatus.UPLOADED
+
+            if check_file_size:
+                if remote_file.size == file.stat().st_size:
+                    file_status[file] = FileVerificationStatus.UPLOADED
+                else:
+                    file_status[file] = FileVerificationStatus.MISMATCHED_SIZE
+
+            if file_status[file] != FileVerificationStatus.UPLOADED:
+                continue  # abort if we already found a mismatch
+
+            if check_file_hash:
+                if remote_file.hash is None:
+                    file_status[file] = FileVerificationStatus.COMPUTING_HASH
+                elif remote_file.hash == b64_md5(file):
+                    file_status[file] = FileVerificationStatus.UPLOADED
+                else:
+                    file_status[file] = FileVerificationStatus.MISMATCHED_HASH
+
         else:
             file_status[file] = FileVerificationStatus.UNKNOWN
+
     return file_status
 
 

kleinkram/models.py

@@ -77,9 +77,10 @@ class File:
 
 # this is the file state for the verify command
 class FileVerificationStatus(str, Enum):
-    UPLAODED = "uploaded"
+    UPLOADED = "uploaded"
     UPLOADING = "uploading"
     COMPUTING_HASH = "computing hash"
     MISSING = "missing"
     MISMATCHED_HASH = "hash mismatch"
+    MISMATCHED_SIZE = "size mismatch"
     UNKNOWN = "unknown"

kleinkram/printing.py

@@ -38,10 +38,11 @@ FILE_STATE_COLOR = {
 
 
 FILE_VERIFICATION_STATUS_STYLES = {
-    FileVerificationStatus.UPLAODED: "green",
+    FileVerificationStatus.UPLOADED: "green",
     FileVerificationStatus.UPLOADING: "yellow",
     FileVerificationStatus.MISSING: "yellow",
     FileVerificationStatus.MISMATCHED_HASH: "red",
+    FileVerificationStatus.MISMATCHED_SIZE: "red",
     FileVerificationStatus.UNKNOWN: "gray",
     FileVerificationStatus.COMPUTING_HASH: "purple",
 }
@@ -296,7 +297,7 @@ def print_file_verification_status(
     else:
         for path, status in file_status.items():
             stream = (
-                sys.stdout if status == FileVerificationStatus.UPLAODED else sys.stderr
+                sys.stdout if status == FileVerificationStatus.UPLOADED else sys.stderr
             )
             print(path, file=stream, flush=True)
 

kleinkram/wrappers.py

@@ -9,7 +9,7 @@ conversion to the internal representation
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Collection
+from typing import Collection, Any
 from typing import Dict
 from typing import List
 from typing import Literal
@@ -39,6 +39,11 @@ def _args_to_project_query(
     project_names: Optional[Sequence[str]] = None,
     project_ids: Optional[Sequence[IdLike]] = None,
 ) -> ProjectQuery:
+
+    # verify types of passed arguments
+    _verify_string_sequence("project_names", project_names)
+    _verify_sequence("project_ids", project_ids)
+
     return ProjectQuery(
         ids=[parse_uuid_like(_id) for _id in project_ids or []],
         patterns=list(project_names or []),
@@ -51,6 +56,13 @@ def _args_to_mission_query(
     project_names: Optional[Sequence[str]] = None,
     project_ids: Optional[Sequence[IdLike]] = None,
 ) -> MissionQuery:
+
+    # verify types of passed arguments
+    _verify_string_sequence("mission_names", mission_names)
+    _verify_sequence("mission_ids", mission_ids)
+    _verify_string_sequence("project_names", project_names)
+    _verify_sequence("project_ids", project_ids)
+
     return MissionQuery(
         ids=[parse_uuid_like(_id) for _id in mission_ids or []],
         patterns=list(mission_names or []),
@@ -60,6 +72,27 @@ def _args_to_mission_query(
     )
 
 
+def _verify_sequence(arg_name: str, arg_value: Optional[Sequence[Any]]) -> None:
+    """Verifies that an argument is either None, or a sequence."""
+    if arg_value is not None:
+        if not isinstance(arg_value, Sequence):
+            raise TypeError(f"{arg_name} must be a Sequence, None, or empty array.")
+
+
+def _verify_string_sequence(arg_name: str, arg_value: Optional[Sequence[Any]]) -> None:
+    """Verifies that an argument is either None, an empty sequence, or a sequence of strings."""
+    if arg_value is not None:
+        if not isinstance(arg_value, Sequence):
+            raise TypeError(f"{arg_name} must be a Sequence, None, or empty array.")
+        if isinstance(arg_value, str):
+            raise TypeError(
+                f"{arg_name} cannot be a string, but a sequence of strings."
+            )
+        for item in arg_value:
+            if not isinstance(item, str):
+                raise TypeError(f"{arg_name} must contain strings only.")
+
+
 def _args_to_file_query(
     file_names: Optional[Sequence[str]] = None,
     file_ids: Optional[Sequence[IdLike]] = None,
@@ -68,6 +101,15 @@ def _args_to_file_query(
     project_names: Optional[Sequence[str]] = None,
     project_ids: Optional[Sequence[IdLike]] = None,
 ) -> FileQuery:
+
+    # verify types of passed arguments
+    _verify_string_sequence("file_names", file_names)
+    _verify_sequence("file_ids", file_ids)
+    _verify_string_sequence("mission_names", mission_names)
+    _verify_sequence("mission_ids", mission_ids)
+    _verify_string_sequence("project_names", project_names)
+    _verify_sequence("project_ids", project_ids)
+
     return FileQuery(
         ids=[parse_uuid_like(_id) for _id in file_ids or []],
         patterns=list(file_names or []),
@@ -288,6 +330,9 @@ def verify(
         project_names=singleton_list(project_name),
         project_ids=singleton_list(project_id),
     )
+
+    _verify_string_sequence("files", files)
+
     return kleinkram.core.verify(
         client=AuthenticatedClient(),
         query=query,

{kleinkram-0.43.6.dev20250401131403.dist-info → kleinkram-0.44.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kleinkram
-Version: 0.43.6.dev20250401131403
+Version: 0.44.0
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch

{kleinkram-0.43.6.dev20250401131403.dist-info → kleinkram-0.44.0.dist-info}/RECORD

@@ -3,31 +3,31 @@ kleinkram/__main__.py,sha256=B9RiZxfO4jpCmWPUHyKJ7_EoZlEG4sPpH-nz7T_YhhQ,125
 kleinkram/_version.py,sha256=QYJyRTcqFcJj4qWYpqs7WcoOP6jxDMqyvxLY-cD6KcE,129
 kleinkram/auth.py,sha256=XD_rHOyJmYYfO7QJf3TLYH5qXA22gXGWi7PT3jujlVs,2968
 kleinkram/config.py,sha256=nx6uSM5nLP4SKe8b9VAx4KDtCCwtyshXmzbEJcUwpsY,7411
-kleinkram/core.py,sha256=kD2aqN_VhlUGZV4ocij8M_VZCQwFsCf42mJmdupRQVU,8678
+kleinkram/core.py,sha256=COJcUx8tP6EQIyg4OiVprJfQUCMbiq2QGB9xZ2CPUCo,9621
 kleinkram/errors.py,sha256=qa98YvhDbLqX60P8bcMcFmHy4HxgYNlSROXud8Kj-P4,965
 kleinkram/main.py,sha256=BTE0mZN__xd46wBhFi6iBlK9eGGQvJ1LdUMsbnysLi0,172
-kleinkram/models.py,sha256=8nJlPrKVLSmehspeuQSFV6nUo76JzehUn6KIZYH1xy4,1832
-kleinkram/printing.py,sha256=_vIjs-lPVgv21ER5q5iYtx44OTs5y8wyd32w27WqocM,12161
+kleinkram/models.py,sha256=MkWdGIWuCSnyg45DbdEUhtGIprwPYwKppYumHkRAS18,1870
+kleinkram/printing.py,sha256=q29iYGkBUCmeJNzNacLHRP-8BqF2UhoXirwX7ja7a1k,12212
 kleinkram/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kleinkram/types.py,sha256=nfDjj8TB1Jn5vqO0Xg6qhLOuKom9DDhe62BrngqnVGM,185
 kleinkram/utils.py,sha256=AsKZxEGStn03L2tqqiMVCQCrDyl8HhwOfpa3no4dfYc,6767
-kleinkram/wrappers.py,sha256=4xXU43eNnvMG2sssU330MmTLSSRdurOpnZ-zNGOGmt0,11342
+kleinkram/wrappers.py,sha256=ZScoEov5Q6D2rvaJJ8E-4f58P_NGWrGc9mRPYxSqOC0,13127
 kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kleinkram/api/client.py,sha256=VwuT97_WdbDpcVGwMXB0fRnUoQnUSf7BOP5eXUFokfI,5932
 kleinkram/api/deser.py,sha256=xRpYUFKZ0Luoo7XyAtYblJvprmpjNSZOiFVnFKmOzcM,4819
-kleinkram/api/file_transfer.py,sha256=smzT_gQjmJHeVYdlfmpCH6CRS_Swx-Kp-3Q8lvTNrqQ,17452
+kleinkram/api/file_transfer.py,sha256=9YLEpbR77LTmK5YKPGC1Eil1_HnyuXOdW1Q1bRd0Wnk,18053
 kleinkram/api/pagination.py,sha256=P_zPsBKlMWkmAv-YfUNHaGW-XLB_4U8BDMrKyiDFIXk,1370
 kleinkram/api/query.py,sha256=9Exi4hJR7Ml38_zjAcOvSEoIAxZLlpM6QwwzO9fs5Gk,3293
 kleinkram/api/routes.py,sha256=x0IfzQO5RAC3rohDSpdUNlVOWl221gh4e0fbMKfGrQA,12251
 kleinkram/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kleinkram/cli/_download.py,sha256=H4YlXJkZE4Md02nzgrO_i8Hsm4ZIejPsxBEKkcn4KHs,2371
+kleinkram/cli/_download.py,sha256=e0fDyp_CFOdbKIUGKmtITvAVINa6STYJk5w5QlElXSs,2394
 kleinkram/cli/_endpoint.py,sha256=oY0p4bnuHLEDJCXtTmir4AHswcKAygZ8I4IWC3RFcKc,1796
 kleinkram/cli/_file.py,sha256=Q2fLDdUyfHFmdGC6wIxMqgEl0F76qszhzWJrRV5rTBM,2973
 kleinkram/cli/_list.py,sha256=5gI3aIUeKC0_eWPQqdFXSBBFvpkTTJSm31TamHa197c,3090
 kleinkram/cli/_mission.py,sha256=zDFnOozOFckpuREFgIPt1IzG5q3b1bsNxYlWQoHoz5A,5301
 kleinkram/cli/_project.py,sha256=N0C96NC_onCEwTteYp2wgkkwkdJt-1q43LFdqNXfjC8,3398
 kleinkram/cli/_upload.py,sha256=gOhbjbmqhmwW7p6bWlSvI53vLHvBFO9QqD1kdU92I2k,2813
-kleinkram/cli/_verify.py,sha256=0ABVa4U_WzaV36ClR8NsOIG7KAMRlnFmsbtnHhbWVj4,1742
+kleinkram/cli/_verify.py,sha256=n9QThY0JnqaIqw6udYXdRQGcpUl2lIbFXGQIgpTnDPE,2112
 kleinkram/cli/app.py,sha256=m2qq4z95QllvXnxh3koPp0kq06I5R9etsJV8qSV8TMk,7037
 kleinkram/cli/error_handling.py,sha256=wK3tzeKVSrZm-xmiyzGLnGT2E4TRpyxhaak6GWGP7P8,1921
 testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,7 +35,7 @@ testing/backend_fixtures.py,sha256=t5QWwyezHUhxxAlbUuE_eFmpyRaGbnWNNcGPwrO17JM,1
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=5MLYQOtQoXWl0TRkYntYKNdqpd4hl9m0XTRi5OXanYI,104
 tests/test_config.py,sha256=uvVh1iUSCoZc8YioxS_GjF-J7m4JE2z4XUbwK_4wDV0,5855
-tests/test_core.py,sha256=9oAb-jxWlRI6EzMwe-oStUFpK4R6ad5vzy95gZgAeuk,5623
+tests/test_core.py,sha256=tVvmC4yBkE00VspoowhfdNAUiBsmkIJQodaC3jjUqqM,5623
 tests/test_end_to_end.py,sha256=0W5pUES5hek-pXq4NZtpPZqKTORkGCRsDv5_D3rDMjY,3372
 tests/test_error_handling.py,sha256=qPSMKF1qsAHyUME0-krxbIrk38iGKkhAyAah-KwN4NE,1300
 tests/test_fixtures.py,sha256=UlPmGbEsGvrDPsaStGMRjNvrVPGjCqOB0RMfLJq2VRA,1071
@@ -43,8 +43,8 @@ tests/test_printing.py,sha256=Jz1AjqmqBRjp1JLm6H1oVJyvGaMPlahVXdKnd7UDQFc,2231
 tests/test_query.py,sha256=fExmCKXLA7-9j2S2sF_sbvRX_2s6Cp3a7OTcqE25q9g,3864
 tests/test_utils.py,sha256=eUBYrn3xrcgcaxm1X4fqZaX4tRvkbI6rh6BUbNbu9T0,4784
 tests/test_wrappers.py,sha256=TbcTyO2L7fslbzgfDdcVZkencxNQ8cGPZm_iB6c9d6Q,2673
-kleinkram-0.43.6.dev20250401131403.dist-info/METADATA,sha256=GuE9LGJ6Dxb3o4PW7HgxSMgr1Qc5srdMLozgwI0RG80,2825
-kleinkram-0.43.6.dev20250401131403.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kleinkram-0.43.6.dev20250401131403.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
-kleinkram-0.43.6.dev20250401131403.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
-kleinkram-0.43.6.dev20250401131403.dist-info/RECORD,,
+kleinkram-0.44.0.dist-info/METADATA,sha256=1d3gNavFGPfJcAAgXyknZYQZ1zjpVcpI1lXFlxmR12A,2807
+kleinkram-0.44.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kleinkram-0.44.0.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
+kleinkram-0.44.0.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
+kleinkram-0.44.0.dist-info/RECORD,,

tests/test_core.py

@@ -167,7 +167,7 @@ def test_verify(mission):
     )
 
     assert all(
-        status == FileVerificationStatus.UPLAODED for status in verify_status.values()
+        status == FileVerificationStatus.UPLOADED for status in verify_status.values()
     )