grz-cli 0.2.0__tar.gz → 0.4.0__tar.gz

grz_cli-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 GRZ/GHGA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

grz_cli-0.4.0/MANIFEST.in

@@ -0,0 +1,2 @@
+graft src
+include LICENSE

{grz_cli-0.2.0/src/grz_cli.egg-info → grz_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.2.0
+Version: 0.4.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License: MIT
@@ -11,7 +11,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
-Requires-Dist: boto3<2,>=1.35.39
+Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
 Requires-Dist: jsonschema<5,>=4.23.0
@@ -19,6 +19,8 @@ Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: tqdm<5,>=4.66.5
 Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
+Requires-Dist: grz-pydantic-models>=1.2.1
+Requires-Dist: pysam==0.23.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -253,6 +255,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.2.0 → grz_cli-0.4.0}/README.md

@@ -223,6 +223,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.2.0 → grz_cli-0.4.0}/pyproject.toml

@@ -4,7 +4,6 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "grz-cli"
-version = "0.2.0"
 description = "Tool for validation, encryption and upload of MV submissions to GDCs."
 readme = "README.md"
 license = { text = "MIT" }
@@ -15,8 +14,9 @@ authors = [
     { name = "Till Hartmann", email = "till.hartmann@bih-charite.de" },
     { name = "Thomas Sell", email = "thomas.sell@bih-charite.de" },
 ]
+dynamic = ["version"]
 dependencies = [
-    "boto3 >=1.35.39,<2",
+    "boto3 >=1.36,<2",
     "click >=8.1.7,<9",
     "crypt4gh >=1.7,<2",
     "jsonschema >=4.23.0,<5",
@@ -24,6 +24,8 @@ dependencies = [
     "tqdm >=4.66.5,<5",
     "pydantic >=2.9.2,<2.10",
     "platformdirs >=4.3.6,<5",
+    "grz-pydantic-models >=1.2.1",
+    "pysam ==0.23.*"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -33,6 +35,10 @@ classifiers = [
 requires-python = ">=3.12"
 keywords = ["GRZ", "GDC", "S3"]
 
+
+[tool.setuptools.dynamic]
+version = { attr = "grz_cli.__version__" }
+
 [project.urls]
 "Homepage" = "https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli"
 
@@ -193,6 +199,10 @@ python_version = "3.12"
 plugins = "pydantic.mypy"
 packages = "src"
 
+[[tool.mypy.overrides]]
+module = ["grz_pydantic_models.*"]
+follow_untyped_imports = true
+
 [tool.pytest.ini_options]
 addopts = [
     "--import-mode=importlib",

grz_cli-0.4.0/src/grz_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

grz_cli-0.4.0/src/grz_cli/bam_validation.py

@@ -0,0 +1,39 @@
+"""
+Validation methods for BAM files
+
+e.g. warn on headers present, which can contain personally identifying
+information
+"""
+
+import logging
+from collections.abc import Generator
+from os import PathLike
+
+import pysam
+
+log = logging.getLogger(__name__)
+
+
+def validate_bam(bam_path: str | PathLike) -> Generator[str]:
+    """Check if a BAM file is valid for submission"""
+    errors: list[str] = []
+
+    # disable pysam's SQ header presence enforcement (check_sq)
+    #  these are used to optionally sort alignments by a custom contig order,
+    #  which is unnecessary in unmapped BAM files
+    bam_file = pysam.AlignmentFile(str(bam_path), mode="rb", check_sq=False)
+
+    header = bam_file.header.to_dict()
+    # ignore the HD key as it is almost always present and extremely
+    # unlikely to accidentally contain identifying information
+    concerning_keys = header.keys() - {"HD"}
+    if concerning_keys:
+        log.warning(f"Detected a header in BAM file '{bam_path}', ensure it contains no private information:")
+        for key in concerning_keys:
+            for line in header[key]:
+                log.warning(f"@{key}")
+                for tag, value in line.items():
+                    log.warning(f" {tag}: {value}")
+        log.warning("--- End of potentially-concerning BAM header information ---")
+
+    yield from errors

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/cli.py

@@ -11,6 +11,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 import click
+import grz_pydantic_models.submission.metadata
 import platformdirs
 import yaml
 
@@ -81,7 +82,7 @@ submission_id = click.option(
     required=True,
     type=str,
     metavar="STRING",
-    help="S3 submission prefix (corresponds to the tanG of a submission)",
+    help="S3 submission ID",
 )
 
 output_dir = click.option(
@@ -108,7 +109,11 @@ class OrderedGroup(click.Group):
     cls=OrderedGroup,
     help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
 )
-@click.version_option(version=importlib.metadata.version("grz-cli"), prog_name="grz-cli")
+@click.version_option(
+    version=importlib.metadata.version("grz-cli"),
+    prog_name="grz-cli",
+    message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
+)
 @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
 @click.option(
     "--log-level",
@@ -270,7 +275,8 @@ def upload(
         encrypted_files_dir=submission_dir / "encrypted_files",
         threads=threads,
     )
-    worker_inst.upload(config)
+    # output the generated submission ID
+    print(worker_inst.upload(config))
 
     log.info("Upload finished!")
 

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/constants.py

@@ -18,7 +18,6 @@ LOGGING_CONFIG = {
             "level": "INFO",
             "formatter": "standard",
             "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",  # Default is stderr
         },
     },
     "loggers": {

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/download.py

@@ -73,8 +73,11 @@ class S3BotoDownloadWorker:
                 return string
 
         # configure proxies if proxy_url is defined
-        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
+        proxy_url = self._config.s3_options.proxy_url
+        config = Boto3Config(
+            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
+            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
+        )
 
         # Initialize S3 client for uploading
         self._s3_client: boto3.session.Session.client = boto3_client(

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/fastq_validation.py

@@ -64,40 +64,58 @@ def open_fastq(file_path: str | PathLike, progress=True) -> Generator[TextIO, No
             yield typing.cast(TextIO, handle)
 
 
-def calculate_fastq_stats(file_path) -> tuple[int, set[int]]:
+def calculate_fastq_stats(file_path, expected_read_length: int | None = None) -> tuple[int, int | None]:
     """
     Calculate line number and read lengths in FASTQ file.
     :param file_path: Path to the FASTQ file
+    :param expected_read_length: Expected read length (None if not known)
     :return: tuple with the following values:
-      - number of lines in the file
-      - Set of observed read lengths
+      - Number of lines in the file
+      - Observed read length
     """
-    read_lengths = set()
-
     with open_fastq(file_path) as f:
         for line_number, line in enumerate(f):
             if (line_number % 4) == 1:
                 # Sequence lines are every 4th line starting from the 2nd
-                read_lengths.add(len(line.strip()))
+                # Check if the read length is consistent
+                read_length = len(line.strip())
+                if expected_read_length is None:
+                    expected_read_length = read_length
+                elif read_length != expected_read_length:
+                    raise ValueError(
+                        f"Read length mismatch at line {line_number + 1}: "
+                        f"expected {expected_read_length}, found {read_length}"
+                    )
 
     return (
         line_number + 1,  # enumerate starts indexing at 0
-        read_lengths,
+        expected_read_length,
     )
 
 
-def validate_fastq_file(fastq_file: str | PathLike) -> tuple[int, set[int], list[str]]:
+def validate_fastq_file(
+    fastq_file: str | PathLike, expected_read_length: int | None = None
+) -> tuple[int, int, list[str]] | tuple[int, int | None, list[str]]:
     """
     Validates a FASTQ file.
 
     :param fastq_file: Path to the fastq file
+    :param expected_read_length: Expected read length (None if not known)
     :return: Tuple with the following fields:
       - number of lines
       - set of observed read lengths
       - list of errors found
     """
-    num_lines, read_lengths = calculate_fastq_stats(fastq_file)
     errors = []
+    try:
+        if expected_read_length is not None:
+            log.debug("%s: expecting read length of %s", fastq_file, expected_read_length)
+
+        # Calculate the number of lines and read lengths
+        num_lines, read_length = calculate_fastq_stats(fastq_file, expected_read_length=expected_read_length)
+    except ValueError as e:
+        errors.append(f"{fastq_file}: {e}")
+        return -1, -1, errors
 
     # Check if the number of lines in a FASTQ file is a multiple of 4.
     if num_lines % 4 != 0:
@@ -105,40 +123,39 @@ def validate_fastq_file(fastq_file: str | PathLike) -> tuple[int, set[int], list
     else:
         log.debug("%s: %s lines", fastq_file, num_lines)
 
-    # Check if all reads in the FASTQ file have the same length.
-    if len(read_lengths) != 1:
-        errors.append(
-            f"{fastq_file}: Inconsistent read lengths! "
-            f"Found reads with following lengths: {', '.join([str(c) for c in read_lengths])}"
-        )
-    else:
-        log.debug("%s: read length: %s", fastq_file, next(iter(read_lengths)))
-
-    return num_lines, read_lengths, errors
+    return num_lines, read_length, errors
 
 
-def validate_single_end_reads(fastq_file: str | PathLike) -> Generator[str]:
+def validate_single_end_reads(fastq_file: str | PathLike, expected_read_length: int | None = None) -> Generator[str]:
     """
     Validate a single-end FASTQ file.
 
     :param fastq_file: Path to the FASTQ file
+    :param expected_read_length: Expected read length (None if not known)
     :return: Generator of errors, if any.
     """
-    num_lines, read_lengths, errors = validate_fastq_file(fastq_file)
+    num_lines, read_lengths, errors = validate_fastq_file(fastq_file, expected_read_length=expected_read_length)
     yield from errors
 
 
-def validate_paired_end_reads(fastq_file1: str | PathLike, fastq_file2: str | PathLike) -> Generator[str]:
+def validate_paired_end_reads(
+    fastq_file1: str | PathLike, fastq_file2: str | PathLike, expected_read_length: int | None = None
+) -> Generator[str]:
     """
     Validate two paired-end FASTQ files.
 
     :param fastq_file1: Path to the first FASTQ file (Read 1)
     :param fastq_file2: Path to the second FASTQ file (Read 2)
+    :param expected_read_length: Expected read length (None if not known)
     :return: Generator of errors, if any.
     """
-    num_lines_file1, read_lengths_file1, errors_file1 = validate_fastq_file(fastq_file1)
+    num_lines_file1, read_lengths_file1, errors_file1 = validate_fastq_file(
+        fastq_file1, expected_read_length=expected_read_length
+    )
     yield from errors_file1
-    num_lines_file2, read_lengths_file2, errors_file2 = validate_fastq_file(fastq_file2)
+    num_lines_file2, read_lengths_file2, errors_file2 = validate_fastq_file(
+        fastq_file2, expected_read_length=expected_read_length
+    )
     yield from errors_file2
 
     if num_lines_file1 != num_lines_file2:

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/models/config.py

@@ -67,6 +67,24 @@ class S3Options(StrictBaseModel):
     The proxy URL for S3 operations (optional).
     """
 
+    request_checksum_calculation: str | None = None
+    """
+    Whether to calculate checksums for S3 request payloads (optional).
+    Valid values are:
+
+    * ``when_supported`` -- When set, a checksum will be calculated for
+      all request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or a
+      ``requestAlgorithmMember`` is modeled.
+
+    * ``when_required`` -- When set, a checksum will only be calculated
+      for request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or where a
+      ``requestAlgorithmMember`` is modeled and supplied.
+
+    Defaults to None.
+    """
+
 
 FilePath = Annotated[Path, AfterValidator(lambda v: v.expanduser()), PathType("file")]
 

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/parser.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 from collections.abc import Generator
@@ -9,13 +10,8 @@ from itertools import groupby
 from os import PathLike
 from pathlib import Path
 
-from pydantic import ValidationError
-
-from .download import S3BotoDownloadWorker
-from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
-from .file_operations import Crypt4GH, calculate_sha256
-from .models.config import ConfigModel
-from .models.v1_1_1.metadata import (
+from grz_pydantic_models.submission.metadata.v1 import (
+    ChecksumType,
     File,
     FileType,
     GrzSubmissionMetadata,
@@ -23,7 +19,14 @@ from .models.v1_1_1.metadata import (
     SequenceData,
     SequencingLayout,
 )
-from .models.v1_1_1.metadata import File as SubmissionFileMetadata
+from grz_pydantic_models.submission.metadata.v1 import File as SubmissionFileMetadata
+from pydantic import ValidationError
+
+from .bam_validation import validate_bam
+from .download import S3BotoDownloadWorker
+from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
+from .file_operations import Crypt4GH, calculate_sha256
+from .models.config import ConfigModel
 from .progress_logging import FileProgressLogger
 from .states import DecryptionState, EncryptionState, ValidationState
 from .upload import S3BotoUploadWorker
@@ -168,6 +171,51 @@ class Submission:
 
         return retval
 
+    @staticmethod
+    def validate_file_data(metadata: File, local_file_path: Path) -> Generator[str]:
+        """
+        Validates whether the provided file matches this metadata.
+
+        :param metadata: Metadata model object
+        :param local_file_path: Path to the actual file (resolved if symlinked)
+        :return: Generator of errors
+        """
+        # Resolve file path
+        local_file_path = local_file_path.resolve()
+
+        # Check if path exists
+        if not local_file_path.exists():
+            yield f"{str(metadata.file_path)} does not exist!"
+            # Return here as following tests cannot work
+            return
+
+        # Check if path is a file
+        if not local_file_path.is_file():
+            yield f"{str(metadata.file_path)} is not a file!"
+            # Return here as following tests cannot work
+            return
+
+        # Check if the checksum is correct
+        if metadata.checksum_type == "sha256":
+            calculated_checksum = calculate_sha256(local_file_path)
+            if metadata.file_checksum != calculated_checksum:
+                yield (
+                    f"{str(metadata.file_path)}: Checksum mismatch! "
+                    f"Expected: '{metadata.file_checksum}', calculated: '{calculated_checksum}'."
+                )
+        else:
+            yield (
+                f"{str(metadata.file_path)}: Unsupported checksum type: {metadata.checksum_type}. "
+                f"Supported types: {[e.value for e in ChecksumType]}"
+            )
+
+        # Check file size
+        if metadata.file_size_in_bytes != local_file_path.stat().st_size:
+            yield (
+                f"{str(metadata.file_path)}: File size mismatch! "
+                f"Expected: '{metadata.file_size_in_bytes}', observed: '{local_file_path.stat().st_size}'."
+            )
+
     def validate_checksums(self, progress_log_file: str | PathLike) -> Generator[str]:
         """
         Validates the checksum of the files against the metadata and prints the errors.
@@ -185,7 +233,7 @@ class Submission:
             self.__log.debug("Validating '%s'...", str(local_file_path))
 
             # validate the file
-            errors = list(file_metadata.validate_data(local_file_path))
+            errors = list(self.validate_file_data(file_metadata, local_file_path))
             validation_passed = len(errors) == 0
 
             # return log state
@@ -219,12 +267,16 @@ class Submission:
         def find_fastq_files(sequence_data: SequenceData) -> list[File]:
             return [f for f in sequence_data.files if f.file_type == FileType.fastq]
 
+        def find_bam_files(sequence_data: SequenceData) -> list[File]:
+            return [f for f in sequence_data.files if f.file_type == FileType.bam]
+
         for donor in self.metadata.content.donors:
             for lab_data in donor.lab_data:
                 sequencing_layout = lab_data.sequencing_layout
                 sequence_data = lab_data.sequence_data
                 # find all FASTQ files
                 fastq_files = find_fastq_files(sequence_data)
+                bam_files = find_bam_files(sequence_data)
 
                 match sequencing_layout:
                     case SequencingLayout.single_end | SequencingLayout.reverse | SequencingLayout.other:
@@ -233,16 +285,45 @@ class Submission:
                     case SequencingLayout.paired_end:
                         yield from self._validate_paired_end(fastq_files, progress_logger)
 
+                yield from self._validate_bams(bam_files, progress_logger)
+
+    def _validate_bams(
+        self,
+        bam_files: list[File],
+        progress_logger: FileProgressLogger[ValidationState],
+    ) -> Generator[str, None, None]:
+        def validate_file(local_file_path, _file_metadata) -> ValidationState:
+            self.__log.debug("Validating '%s'...", str(local_file_path))
+
+            # validate the file
+            errors = list(validate_bam(local_file_path))
+            validation_passed = len(errors) == 0
+
+            # return log state
+            return ValidationState(
+                errors=errors,
+                validation_passed=validation_passed,
+            )
+
+        for bam_file in bam_files:
+            logged_state = progress_logger.get_state(
+                self.files_dir / bam_file.file_path,
+                bam_file,
+                default=validate_file,  # validate the file if the state was not calculated yet
+            )
+            if logged_state:
+                yield from logged_state["errors"]
+
     def _validate_single_end(
         self,
         fastq_files: list[File],
         progress_logger: FileProgressLogger[ValidationState],
     ) -> Generator[str, None, None]:
-        def validate_file(local_file_path, _file_metadata) -> ValidationState:
+        def validate_file(local_file_path, file_metadata: SubmissionFileMetadata) -> ValidationState:
             self.__log.debug("Validating '%s'...", str(local_file_path))
 
             # validate the file
-            errors = list(validate_single_end_reads(local_file_path))
+            errors = list(validate_single_end_reads(local_file_path, expected_read_length=file_metadata.read_length))
             validation_passed = len(errors) == 0
 
             # return log state
@@ -293,6 +374,7 @@ class Submission:
                         validate_paired_end_reads(
                             local_fastq_r1_path,  # fastq R1
                             local_fastq_r2_path,  # fastq R2
+                            expected_read_length=fastq_r1.read_length,  # fastq R1 and R2 should have the same read length
                         )
                     )
                     validation_passed = len(errors) == 0
@@ -447,13 +529,26 @@ class EncryptedSubmission:
 
         return retval
 
+    _submission_id: str | None = None
+
+    @property
+    def submission_id(self) -> str:
+        """ID to refer to an individual submission to a GRZ"""
+        if self._submission_id is None:
+            # generate a submission ID once
+            submitter_id = self.metadata.content.submission.submitter_id
+            submission_date = self.metadata.content.submission.submission_date
+            # use first 8 characters of SHA256 hash of transaction ID to virtually prevent collisions
+            suffix = hashlib.sha256(self.metadata.transaction_id.encode("utf-8")).hexdigest()[:8]
+            self._submission_id = f"{submitter_id}_{submission_date}_{suffix}"
+
+        return self._submission_id
+
     def get_metadata_file_path_and_object_id(self) -> tuple[Path, str]:
         """
         :return: tuple with the `local_file_path` and s3_object_id of the metadata file
         """
-        return Path(self.metadata.file_path), str(
-            Path(self.metadata.transaction_id) / "metadata" / self.metadata.file_path.name
-        )
+        return Path(self.metadata.file_path), str(Path(self.submission_id) / "metadata" / self.metadata.file_path.name)
 
     def get_encrypted_files_and_object_id(self) -> dict[Path, str]:
         """
@@ -462,7 +557,7 @@ class EncryptedSubmission:
         retval = {}
         for local_file_path, file_metadata in self.encrypted_files.items():
             retval[local_file_path] = str(
-                Path(self.metadata.transaction_id) / "files" / self.get_encrypted_file_path(file_metadata.file_path)
+                Path(self.submission_id) / "files" / self.get_encrypted_file_path(file_metadata.file_path)
             )
         return retval
 
@@ -742,10 +837,9 @@ class Worker:
 
         return submission
 
-    def upload(self, config: ConfigModel):
+    def upload(self, config: ConfigModel) -> str:
         """
-        Upload an encrypted submission
-
+        Upload an encrypted submission and return the generated submission ID
         """
         upload_worker = S3BotoUploadWorker(config, status_file_path=self.progress_file_upload)
 
@@ -753,6 +847,8 @@ class Worker:
 
         upload_worker.upload(encrypted_submission)
 
+        return encrypted_submission.submission_id
+
     def download(self, config: ConfigModel, submission_id: str):
         """
         Download an encrypted submission

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/progress_logging.py

@@ -9,8 +9,9 @@ from collections.abc import Callable
 from os import PathLike
 from pathlib import Path
 
+from grz_pydantic_models.submission.metadata.v1 import File as SubmissionFileMetadata
+
 from .file_operations import read_multiple_json
-from .models.v1_1_1.metadata import File as SubmissionFileMetadata
 from .states import State
 
 Index = tuple[str, float, int]
@@ -104,7 +105,8 @@ class FileProgressLogger[T: State]:
         :param file_path: The file path to query for the state.
         :param file_metadata: The metadata of the file to query for the state.
         :param default: Default state to use if the file does not exist.
-            Can be a Callable that takes the file path and the file metadata as input and returns some state.
+            Can be a Callable that takes the file path and the file metadata as input and returns some state:
+            `Callable[[Path, SubmissionFileMetadata], T]`.
 
             The default state gets automatically saved as the state for this file in case there is no stored state.
         :return: A dictionary representing the file's state, or None if the file's state isn't logged.

{grz_cli-0.2.0 → grz_cli-0.4.0}/src/grz_cli/upload.py

@@ -98,8 +98,11 @@ class S3BotoUploadWorker(UploadWorker):
                 return string
 
         # configure proxies if proxy_url is defined
-        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
+        proxy_url = self._config.s3_options.proxy_url
+        config = Boto3Config(
+            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
+            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
+        )
 
         # Initialize S3 client for uploading
         self._s3_client: boto3.session.Session.client = boto3_client(
@@ -130,7 +133,9 @@ class S3BotoUploadWorker(UploadWorker):
             if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
             else MULTIPART_CHUNKSIZE
         )
-        self.__log.debug(f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize / chunksize} chunks")
+        self.__log.debug(
+            f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {math.ceil(filesize / chunksize)} chunk(s)"
+        )
 
         config = TransferConfig(
             multipart_threshold=MULTIPART_THRESHOLD,