grz-cli 0.3.0__tar.gz → 0.4.0__tar.gz

grz_cli-0.4.0/MANIFEST.in

@@ -0,0 +1,2 @@
+graft src
+include LICENSE

{grz_cli-0.3.0/src/grz_cli.egg-info → grz_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.3.0
+Version: 0.4.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License: MIT
@@ -11,7 +11,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
-Requires-Dist: boto3<2,>=1.35.39
+Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
 Requires-Dist: jsonschema<5,>=4.23.0
@@ -19,7 +19,8 @@ Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: tqdm<5,>=4.66.5
 Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
-Requires-Dist: grz-pydantic-models>=1.2.0
+Requires-Dist: grz-pydantic-models>=1.2.1
+Requires-Dist: pysam==0.23.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -254,6 +255,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.3.0 → grz_cli-0.4.0}/README.md

@@ -223,6 +223,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.3.0 → grz_cli-0.4.0}/pyproject.toml

@@ -16,7 +16,7 @@ authors = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "boto3 >=1.35.39,<2",
+    "boto3 >=1.36,<2",
     "click >=8.1.7,<9",
     "crypt4gh >=1.7,<2",
     "jsonschema >=4.23.0,<5",
@@ -24,7 +24,8 @@ dependencies = [
     "tqdm >=4.66.5,<5",
     "pydantic >=2.9.2,<2.10",
     "platformdirs >=4.3.6,<5",
-    "grz-pydantic-models >=1.2.0"
+    "grz-pydantic-models >=1.2.1",
+    "pysam ==0.23.*"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",

grz_cli-0.4.0/src/grz_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

grz_cli-0.4.0/src/grz_cli/bam_validation.py

@@ -0,0 +1,39 @@
+"""
+Validation methods for BAM files
+
+e.g. warn on headers present, which can contain personally identifying
+information
+"""
+
+import logging
+from collections.abc import Generator
+from os import PathLike
+
+import pysam
+
+log = logging.getLogger(__name__)
+
+
+def validate_bam(bam_path: str | PathLike) -> Generator[str]:
+    """Check if a BAM file is valid for submission"""
+    errors: list[str] = []
+
+    # disable pysam's SQ header presence enforcement (check_sq)
+    #  these are used to optionally sort alignments by a custom contig order,
+    #  which is unnecessary in unmapped BAM files
+    bam_file = pysam.AlignmentFile(str(bam_path), mode="rb", check_sq=False)
+
+    header = bam_file.header.to_dict()
+    # ignore the HD key as it is almost always present and extremely
+    # unlikely to accidentally contain identifying information
+    concerning_keys = header.keys() - {"HD"}
+    if concerning_keys:
+        log.warning(f"Detected a header in BAM file '{bam_path}', ensure it contains no private information:")
+        for key in concerning_keys:
+            for line in header[key]:
+                log.warning(f"@{key}")
+                for tag, value in line.items():
+                    log.warning(f" {tag}: {value}")
+        log.warning("--- End of potentially-concerning BAM header information ---")
+
+    yield from errors

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/cli.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 import click
-import grz_pydantic_models
+import grz_pydantic_models.submission.metadata
 import platformdirs
 import yaml
 
@@ -82,7 +82,7 @@ submission_id = click.option(
     required=True,
     type=str,
     metavar="STRING",
-    help="S3 submission prefix (corresponds to the tanG of a submission)",
+    help="S3 submission ID",
 )
 
 output_dir = click.option(
@@ -112,7 +112,7 @@ class OrderedGroup(click.Group):
 @click.version_option(
     version=importlib.metadata.version("grz-cli"),
     prog_name="grz-cli",
-    message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.get_supported_versions())})",
+    message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
 )
 @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
 @click.option(
@@ -275,7 +275,8 @@ def upload(
         encrypted_files_dir=submission_dir / "encrypted_files",
         threads=threads,
     )
-    worker_inst.upload(config)
+    # output the generated submission ID
+    print(worker_inst.upload(config))
 
     log.info("Upload finished!")
 

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/constants.py

@@ -18,7 +18,6 @@ LOGGING_CONFIG = {
             "level": "INFO",
             "formatter": "standard",
             "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",  # Default is stderr
         },
     },
     "loggers": {

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/download.py

@@ -73,8 +73,11 @@ class S3BotoDownloadWorker:
                 return string
 
         # configure proxies if proxy_url is defined
-        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
+        proxy_url = self._config.s3_options.proxy_url
+        config = Boto3Config(
+            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
+            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
+        )
 
         # Initialize S3 client for uploading
         self._s3_client: boto3.session.Session.client = boto3_client(

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/fastq_validation.py

@@ -64,40 +64,58 @@ def open_fastq(file_path: str | PathLike, progress=True) -> Generator[TextIO, No
             yield typing.cast(TextIO, handle)
 
 
-def calculate_fastq_stats(file_path) -> tuple[int, set[int]]:
+def calculate_fastq_stats(file_path, expected_read_length: int | None = None) -> tuple[int, int | None]:
     """
     Calculate line number and read lengths in FASTQ file.
     :param file_path: Path to the FASTQ file
+    :param expected_read_length: Expected read length (None if not known)
     :return: tuple with the following values:
-      - number of lines in the file
-      - Set of observed read lengths
+      - Number of lines in the file
+      - Observed read length
     """
-    read_lengths = set()
-
     with open_fastq(file_path) as f:
         for line_number, line in enumerate(f):
             if (line_number % 4) == 1:
                 # Sequence lines are every 4th line starting from the 2nd
-                read_lengths.add(len(line.strip()))
+                # Check if the read length is consistent
+                read_length = len(line.strip())
+                if expected_read_length is None:
+                    expected_read_length = read_length
+                elif read_length != expected_read_length:
+                    raise ValueError(
+                        f"Read length mismatch at line {line_number + 1}: "
+                        f"expected {expected_read_length}, found {read_length}"
+                    )
 
     return (
         line_number + 1,  # enumerate starts indexing at 0
-        read_lengths,
+        expected_read_length,
     )
 
 
-def validate_fastq_file(fastq_file: str | PathLike) -> tuple[int, set[int], list[str]]:
+def validate_fastq_file(
+    fastq_file: str | PathLike, expected_read_length: int | None = None
+) -> tuple[int, int, list[str]] | tuple[int, int | None, list[str]]:
     """
     Validates a FASTQ file.
 
     :param fastq_file: Path to the fastq file
+    :param expected_read_length: Expected read length (None if not known)
     :return: Tuple with the following fields:
       - number of lines
       - set of observed read lengths
       - list of errors found
     """
-    num_lines, read_lengths = calculate_fastq_stats(fastq_file)
     errors = []
+    try:
+        if expected_read_length is not None:
+            log.debug("%s: expecting read length of %s", fastq_file, expected_read_length)
+
+        # Calculate the number of lines and read lengths
+        num_lines, read_length = calculate_fastq_stats(fastq_file, expected_read_length=expected_read_length)
+    except ValueError as e:
+        errors.append(f"{fastq_file}: {e}")
+        return -1, -1, errors
 
     # Check if the number of lines in a FASTQ file is a multiple of 4.
     if num_lines % 4 != 0:
@@ -105,40 +123,39 @@ def validate_fastq_file(fastq_file: str | PathLike) -> tuple[int, set[int], list
     else:
         log.debug("%s: %s lines", fastq_file, num_lines)
 
-    # Check if all reads in the FASTQ file have the same length.
-    if len(read_lengths) != 1:
-        errors.append(
-            f"{fastq_file}: Inconsistent read lengths! "
-            f"Found reads with following lengths: {', '.join([str(c) for c in read_lengths])}"
-        )
-    else:
-        log.debug("%s: read length: %s", fastq_file, next(iter(read_lengths)))
-
-    return num_lines, read_lengths, errors
+    return num_lines, read_length, errors
 
 
-def validate_single_end_reads(fastq_file: str | PathLike) -> Generator[str]:
+def validate_single_end_reads(fastq_file: str | PathLike, expected_read_length: int | None = None) -> Generator[str]:
     """
     Validate a single-end FASTQ file.
 
     :param fastq_file: Path to the FASTQ file
+    :param expected_read_length: Expected read length (None if not known)
     :return: Generator of errors, if any.
     """
-    num_lines, read_lengths, errors = validate_fastq_file(fastq_file)
+    num_lines, read_lengths, errors = validate_fastq_file(fastq_file, expected_read_length=expected_read_length)
     yield from errors
 
 
-def validate_paired_end_reads(fastq_file1: str | PathLike, fastq_file2: str | PathLike) -> Generator[str]:
+def validate_paired_end_reads(
+    fastq_file1: str | PathLike, fastq_file2: str | PathLike, expected_read_length: int | None = None
+) -> Generator[str]:
     """
     Validate two paired-end FASTQ files.
 
     :param fastq_file1: Path to the first FASTQ file (Read 1)
     :param fastq_file2: Path to the second FASTQ file (Read 2)
+    :param expected_read_length: Expected read length (None if not known)
     :return: Generator of errors, if any.
     """
-    num_lines_file1, read_lengths_file1, errors_file1 = validate_fastq_file(fastq_file1)
+    num_lines_file1, read_lengths_file1, errors_file1 = validate_fastq_file(
+        fastq_file1, expected_read_length=expected_read_length
+    )
     yield from errors_file1
-    num_lines_file2, read_lengths_file2, errors_file2 = validate_fastq_file(fastq_file2)
+    num_lines_file2, read_lengths_file2, errors_file2 = validate_fastq_file(
+        fastq_file2, expected_read_length=expected_read_length
+    )
     yield from errors_file2
 
     if num_lines_file1 != num_lines_file2:

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/models/config.py

@@ -67,6 +67,24 @@ class S3Options(StrictBaseModel):
     The proxy URL for S3 operations (optional).
     """
 
+    request_checksum_calculation: str | None = None
+    """
+    Whether to calculate checksums for S3 request payloads (optional).
+    Valid values are:
+
+    * ``when_supported`` -- When set, a checksum will be calculated for
+      all request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or a
+      ``requestAlgorithmMember`` is modeled.
+
+    * ``when_required`` -- When set, a checksum will only be calculated
+      for request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or where a
+      ``requestAlgorithmMember`` is modeled and supplied.
+
+    Defaults to None.
+    """
+
 
 FilePath = Annotated[Path, AfterValidator(lambda v: v.expanduser()), PathType("file")]
 

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/parser.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 from collections.abc import Generator
@@ -9,7 +10,7 @@ from itertools import groupby
 from os import PathLike
 from pathlib import Path
 
-from grz_pydantic_models.v1_1_1.metadata import (
+from grz_pydantic_models.submission.metadata.v1 import (
     ChecksumType,
     File,
     FileType,
@@ -18,9 +19,10 @@ from grz_pydantic_models.v1_1_1.metadata import (
     SequenceData,
     SequencingLayout,
 )
-from grz_pydantic_models.v1_1_1.metadata import File as SubmissionFileMetadata
+from grz_pydantic_models.submission.metadata.v1 import File as SubmissionFileMetadata
 from pydantic import ValidationError
 
+from .bam_validation import validate_bam
 from .download import S3BotoDownloadWorker
 from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
 from .file_operations import Crypt4GH, calculate_sha256
@@ -265,12 +267,16 @@ class Submission:
         def find_fastq_files(sequence_data: SequenceData) -> list[File]:
             return [f for f in sequence_data.files if f.file_type == FileType.fastq]
 
+        def find_bam_files(sequence_data: SequenceData) -> list[File]:
+            return [f for f in sequence_data.files if f.file_type == FileType.bam]
+
         for donor in self.metadata.content.donors:
             for lab_data in donor.lab_data:
                 sequencing_layout = lab_data.sequencing_layout
                 sequence_data = lab_data.sequence_data
                 # find all FASTQ files
                 fastq_files = find_fastq_files(sequence_data)
+                bam_files = find_bam_files(sequence_data)
 
                 match sequencing_layout:
                     case SequencingLayout.single_end | SequencingLayout.reverse | SequencingLayout.other:
@@ -279,16 +285,45 @@ class Submission:
                     case SequencingLayout.paired_end:
                         yield from self._validate_paired_end(fastq_files, progress_logger)
 
+                yield from self._validate_bams(bam_files, progress_logger)
+
+    def _validate_bams(
+        self,
+        bam_files: list[File],
+        progress_logger: FileProgressLogger[ValidationState],
+    ) -> Generator[str, None, None]:
+        def validate_file(local_file_path, _file_metadata) -> ValidationState:
+            self.__log.debug("Validating '%s'...", str(local_file_path))
+
+            # validate the file
+            errors = list(validate_bam(local_file_path))
+            validation_passed = len(errors) == 0
+
+            # return log state
+            return ValidationState(
+                errors=errors,
+                validation_passed=validation_passed,
+            )
+
+        for bam_file in bam_files:
+            logged_state = progress_logger.get_state(
+                self.files_dir / bam_file.file_path,
+                bam_file,
+                default=validate_file,  # validate the file if the state was not calculated yet
+            )
+            if logged_state:
+                yield from logged_state["errors"]
+
     def _validate_single_end(
         self,
         fastq_files: list[File],
         progress_logger: FileProgressLogger[ValidationState],
     ) -> Generator[str, None, None]:
-        def validate_file(local_file_path, _file_metadata) -> ValidationState:
+        def validate_file(local_file_path, file_metadata: SubmissionFileMetadata) -> ValidationState:
             self.__log.debug("Validating '%s'...", str(local_file_path))
 
             # validate the file
-            errors = list(validate_single_end_reads(local_file_path))
+            errors = list(validate_single_end_reads(local_file_path, expected_read_length=file_metadata.read_length))
             validation_passed = len(errors) == 0
 
             # return log state
@@ -339,6 +374,7 @@ class Submission:
                         validate_paired_end_reads(
                             local_fastq_r1_path,  # fastq R1
                             local_fastq_r2_path,  # fastq R2
+                            expected_read_length=fastq_r1.read_length,  # fastq R1 and R2 should have the same read length
                         )
                     )
                     validation_passed = len(errors) == 0
@@ -493,13 +529,26 @@ class EncryptedSubmission:
 
         return retval
 
+    _submission_id: str | None = None
+
+    @property
+    def submission_id(self) -> str:
+        """ID to refer to an individual submission to a GRZ"""
+        if self._submission_id is None:
+            # generate a submission ID once
+            submitter_id = self.metadata.content.submission.submitter_id
+            submission_date = self.metadata.content.submission.submission_date
+            # use first 8 characters of SHA256 hash of transaction ID to virtually prevent collisions
+            suffix = hashlib.sha256(self.metadata.transaction_id.encode("utf-8")).hexdigest()[:8]
+            self._submission_id = f"{submitter_id}_{submission_date}_{suffix}"
+
+        return self._submission_id
+
     def get_metadata_file_path_and_object_id(self) -> tuple[Path, str]:
         """
         :return: tuple with the `local_file_path` and s3_object_id of the metadata file
         """
-        return Path(self.metadata.file_path), str(
-            Path(self.metadata.transaction_id) / "metadata" / self.metadata.file_path.name
-        )
+        return Path(self.metadata.file_path), str(Path(self.submission_id) / "metadata" / self.metadata.file_path.name)
 
     def get_encrypted_files_and_object_id(self) -> dict[Path, str]:
         """
@@ -508,7 +557,7 @@ class EncryptedSubmission:
         retval = {}
         for local_file_path, file_metadata in self.encrypted_files.items():
             retval[local_file_path] = str(
-                Path(self.metadata.transaction_id) / "files" / self.get_encrypted_file_path(file_metadata.file_path)
+                Path(self.submission_id) / "files" / self.get_encrypted_file_path(file_metadata.file_path)
             )
         return retval
 
@@ -788,10 +837,9 @@ class Worker:
 
         return submission
 
-    def upload(self, config: ConfigModel):
+    def upload(self, config: ConfigModel) -> str:
         """
-        Upload an encrypted submission
-
+        Upload an encrypted submission and return the generated submission ID
         """
         upload_worker = S3BotoUploadWorker(config, status_file_path=self.progress_file_upload)
 
@@ -799,6 +847,8 @@ class Worker:
 
         upload_worker.upload(encrypted_submission)
 
+        return encrypted_submission.submission_id
+
     def download(self, config: ConfigModel, submission_id: str):
         """
         Download an encrypted submission

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/progress_logging.py

@@ -9,7 +9,7 @@ from collections.abc import Callable
 from os import PathLike
 from pathlib import Path
 
-from grz_pydantic_models.v1_1_1.metadata import File as SubmissionFileMetadata
+from grz_pydantic_models.submission.metadata.v1 import File as SubmissionFileMetadata
 
 from .file_operations import read_multiple_json
 from .states import State
@@ -105,7 +105,8 @@ class FileProgressLogger[T: State]:
         :param file_path: The file path to query for the state.
         :param file_metadata: The metadata of the file to query for the state.
         :param default: Default state to use if the file does not exist.
-            Can be a Callable that takes the file path and the file metadata as input and returns some state.
+            Can be a Callable that takes the file path and the file metadata as input and returns some state:
+            `Callable[[Path, SubmissionFileMetadata], T]`.
 
             The default state gets automatically saved as the state for this file in case there is no stored state.
         :return: A dictionary representing the file's state, or None if the file's state isn't logged.

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli/upload.py

@@ -98,8 +98,11 @@ class S3BotoUploadWorker(UploadWorker):
                 return string
 
         # configure proxies if proxy_url is defined
-        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
+        proxy_url = self._config.s3_options.proxy_url
+        config = Boto3Config(
+            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
+            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
+        )
 
         # Initialize S3 client for uploading
         self._s3_client: boto3.session.Session.client = boto3_client(
@@ -130,7 +133,9 @@ class S3BotoUploadWorker(UploadWorker):
             if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
             else MULTIPART_CHUNKSIZE
         )
-        self.__log.debug(f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize / chunksize} chunks")
+        self.__log.debug(
+            f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {math.ceil(filesize / chunksize)} chunk(s)"
+        )
 
         config = TransferConfig(
             multipart_threshold=MULTIPART_THRESHOLD,

{grz_cli-0.3.0 → grz_cli-0.4.0/src/grz_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.3.0
+Version: 0.4.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License: MIT
@@ -11,7 +11,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
-Requires-Dist: boto3<2,>=1.35.39
+Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
 Requires-Dist: jsonschema<5,>=4.23.0
@@ -19,7 +19,8 @@ Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: tqdm<5,>=4.66.5
 Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
-Requires-Dist: grz-pydantic-models>=1.2.0
+Requires-Dist: grz-pydantic-models>=1.2.1
+Requires-Dist: pysam==0.23.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -254,6 +255,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ README.md
 pyproject.toml
 src/__init__.py
 src/grz_cli/__init__.py
+src/grz_cli/bam_validation.py
 src/grz_cli/cli.py
 src/grz_cli/constants.py
 src/grz_cli/download.py
@@ -22,7 +23,7 @@ src/grz_cli.egg-info/requires.txt
 src/grz_cli.egg-info/top_level.txt
 src/grz_cli/models/__init__.py
 src/grz_cli/models/config.py
-src/grz_cli/resources/thresholds.json
+tests/test_bam_validation.py
 tests/test_cli.py
 tests/test_download.py
 tests/test_fastq_validation.py

{grz_cli-0.3.0 → grz_cli-0.4.0}/src/grz_cli.egg-info/requires.txt

@@ -1,4 +1,4 @@
-boto3<2,>=1.35.39
+boto3<2,>=1.36
 click<9,>=8.1.7
 crypt4gh<2,>=1.7
 jsonschema<5,>=4.23.0
@@ -6,7 +6,8 @@ pyyaml<7,>=6.0.2
 tqdm<5,>=4.66.5
 pydantic<2.10,>=2.9.2
 platformdirs<5,>=4.3.6
-grz-pydantic-models>=1.2.0
+grz-pydantic-models>=1.2.1
+pysam==0.23.*
 
 [test]
 pytest<9,>=8.3.3

grz_cli-0.4.0/tests/test_bam_validation.py

@@ -0,0 +1,15 @@
+"""Tests for the bam_validation module."""
+
+import importlib.resources
+
+from grz_cli.bam_validation import validate_bam
+
+from . import resources
+
+
+def test_valid_hifi_bam():
+    """Valid HiFi BAM files should pass validation"""
+    bam_ptr = importlib.resources.files(resources).joinpath("reads", "valid_HiFi.bam")
+    with importlib.resources.as_file(bam_ptr) as bam_path:
+        errors = list(validate_bam(bam_path))
+    assert len(errors) == 0

{grz_cli-0.3.0 → grz_cli-0.4.0}/tests/test_cli.py

@@ -183,7 +183,6 @@ def test_upload_download_submission(
     remote_bucket,
 ):
     submission_dir = Path("tests/mock_files/submissions/valid_submission")
-    transaction_id = "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000"
 
     shutil.copytree(
         submission_dir / "encrypted_files",
@@ -200,7 +199,7 @@ def test_upload_download_submission(
         "--config-file",
         temp_config_file_path,
     ]
-    runner = CliRunner()
+    runner = CliRunner(mix_stderr=False)
     with mock.patch(
         "grz_cli.models.config.S3Options.__getattr__",
         lambda self, name: None if name == "endpoint_url" else AttributeError,
@@ -208,6 +207,9 @@ def test_upload_download_submission(
         result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
+    assert len(result.output) != 0, result.stderr
+
+    submission_id = result.output.strip()
 
     # download
     download_dir = tmpdir_factory.mktemp("submission_download")
@@ -217,7 +219,7 @@ def test_upload_download_submission(
     testargs = [
         "download",
         "--submission-id",
-        transaction_id,
+        submission_id,
         "--output-dir",
         str(download_dir_path),
         "--config-file",

{grz_cli-0.3.0 → grz_cli-0.4.0}/tests/test_fastq_validation.py

@@ -21,7 +21,7 @@ def test_single_end_inconsistent_read_length():
         )
     )
     assert len(errors) == 1
-    assert "Inconsistent read lengths" in errors[0]
+    assert "Read length mismatch" in errors[0]
 
 
 # Test case 3: Paired end, differing line numbers

{grz_cli-0.3.0 → grz_cli-0.4.0}/tests/test_file_progress_logging.py

@@ -5,7 +5,7 @@ import json
 from pathlib import Path
 
 import pytest
-from grz_pydantic_models.v1_1_1.metadata import File as SubmissionFileMetadata
+from grz_pydantic_models.submission.metadata.v1 import File as SubmissionFileMetadata
 
 from grz_cli.file_operations import read_multiple_json
 from grz_cli.progress_logging import FileProgressLogger

{grz_cli-0.3.0 → grz_cli-0.4.0}/tests/test_upload.py

@@ -62,6 +62,7 @@ def test_boto_upload(
 
 
 def test__gather_files_to_upload(encrypted_submission):
+    submission_id = encrypted_submission.submission_id
     metadata_file_path, metadata_s3_object_id = encrypted_submission.get_metadata_file_path_and_object_id()
     gathered_files = encrypted_submission.get_encrypted_files_and_object_id()
     gathered_files[metadata_file_path] = metadata_s3_object_id
@@ -70,47 +71,47 @@ def test__gather_files_to_upload(encrypted_submission):
     expected_files = [
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/target_regions.bed.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/target_regions.bed.c4gh",
+            f"{submission_id}/files/target_regions.bed.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read1.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read1.fastq.gz.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read1.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read2.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read2.fastq.gz.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read2.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.vcf.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.vcf.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_normal.vcf.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read1.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read1.fastq.gz.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read1.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read2.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read2.fastq.gz.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read2.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.vcf.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.vcf.c4gh",
+            f"{submission_id}/files/aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.vcf.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read1.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read1.fastq.gz.c4gh",
+            f"{submission_id}/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read1.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read2.fastq.gz.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read2.fastq.gz.c4gh",
+            f"{submission_id}/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.read2.fastq.gz.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/encrypted_files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.vcf.c4gh",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.vcf.c4gh",
+            f"{submission_id}/files/bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111bbbbbbbb11111111_blood_normal.vcf.c4gh",
         ),
         (
             "tests/mock_files/submissions/valid_submission/metadata/metadata.json",
-            "aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000aaaaaaaa00000000/metadata/metadata.json",
+            f"{submission_id}/metadata/metadata.json",
         ),
     ]
     expected_files = sorted(expected_files)

grz_cli-0.3.0/MANIFEST.in

@@ -1,3 +0,0 @@
-graft src
-include src/grz_cli/resources/thresholds.json
-include LICENSE

grz_cli-0.3.0/src/grz_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"

grz_cli-0.3.0/src/grz_cli/resources/thresholds.json

@@ -1,299 +0,0 @@
-[
-  {
-    "libraryType": "panel",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "panel",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 30,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 20,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "germline-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 30,
-      "readLength": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 30,
-        "fractionBasesAbove": 0.85
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 20,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "panel_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "panel_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 200,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 100,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wes_lr",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs_lr",
-    "sequenceSubtype": "somatic",
-    "genomicStudySubtype": "tumor-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 100,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 30,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs_lr",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "tumor+germline",
-    "thresholds": {
-      "meanDepthOfCoverage": 30,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 20,
-        "fractionAbove": 0.8
-      }
-    }
-  },
-  {
-    "libraryType": "wgs_lr",
-    "sequenceSubtype": "germline",
-    "genomicStudySubtype": "germline-only",
-    "thresholds": {
-      "meanDepthOfCoverage": 30,
-      "fractionBasesAboveQualityThreshold": {
-        "qualityThreshold": 20,
-        "fractionBasesAbove": 0.8
-      },
-      "targetedRegionsAboveMinCoverage": {
-        "minCoverage": 20,
-        "fractionAbove": 0.8
-      }
-    }
-  }
-]