grz-cli 0.1.2__tar.gz → 0.1.3__tar.gz

{grz_cli-0.1.2 → grz_cli-0.1.3}/MANIFEST.in

@@ -1 +1,2 @@
 include src/grz_cli/resources/grz-schema.json
+include src/grz_cli/resources/thresholds.json

{grz_cli-0.1.2/src/grz_cli.egg-info → grz_cli-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: grz-cli
-Version: 0.1.2
+Version: 0.1.3
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License: MIT

{grz_cli-0.1.2 → grz_cli-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "grz-cli"
-version = "0.1.2"
+version = "0.1.3"
 description = "Tool for validation, encryption and upload of MV submissions to GDCs."
 readme = "README.md"
 license = { text = "MIT" }
@@ -76,7 +76,7 @@ exclude = [
     "dist",
     "out"
 ]
-line-length = 88
+line-length = 120
 src = [
     "src",
     "tests"
@@ -149,7 +149,7 @@ convention = "pep257"
 
 [tool.tox]
 requires = ["tox>=4.23"]
-env_list = ["format-check", "lints", "mypy", "3.12"]
+env_list = ["format-check", "lints", "mypy", "3.12", "3.13"]
 allowlist_externals = ["pytest", "ruff"]
 isolated_build = true
 
@@ -188,3 +188,7 @@ python_version = "3.12"
 plugins = "pydantic.mypy"
 packages = "src"
 
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+]

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/cli.py

@@ -48,9 +48,7 @@ DIR_RW_C = click.Path(
     writable=True,
     resolve_path=True,
 )
-FILE_R_E = click.Path(
-    exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-)
+FILE_R_E = click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True)
 
 submission_dir = click.option(
     "--submission-dir",

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/download.py

@@ -74,10 +74,7 @@ class S3BotoDownloadWorker:
 
         # configure proxies if proxy_url is defined
         proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        if proxy_url is not None:
-            config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url})
-        else:
-            config = None
+        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
 
         # Initialize S3 client for uploading
         self._s3_client: boto3.session.Session.client = boto3_client(
@@ -149,9 +146,7 @@ class S3BotoDownloadWorker:
             if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
             else MULTIPART_CHUNKSIZE
         )
-        self.__log.debug(
-            f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize/chunksize} chunks"
-        )
+        self.__log.debug(f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize/chunksize} chunks")
 
         config = TransferConfig(
             multipart_threshold=MULTIPART_THRESHOLD,
@@ -160,9 +155,7 @@ class S3BotoDownloadWorker:
         )
 
         transfer = S3Transfer(self._s3_client, config)
-        progress_bar = tqdm(
-            total=filesize, unit="B", unit_scale=True, unit_divisor=1024
-        )
+        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024)
         transfer.download_file(
             self._config.s3_options.bucket,
             s3_object_id,
@@ -192,15 +185,11 @@ class S3BotoDownloadWorker:
                     file_path = Path(file_key).relative_to(encrypted_files_key_prefix)
                     full_path = encrypted_submission.encrypted_files_dir / file_path
                     if full_path not in encrypted_submission.encrypted_files:
-                        raise DownloadError(
-                            f"File {file_path} not listed in metadata.json"
-                        )
+                        raise DownloadError(f"File {file_path} not listed in metadata.json")
                     file_metadata = encrypted_submission.encrypted_files[full_path]
                     logged_state = progress_logger.get_state(full_path, file_metadata)
 
-                    if (logged_state is None) or not logged_state.get(
-                        "download_successful", False
-                    ):
+                    if (logged_state is None) or not logged_state.get("download_successful", False):
                         self.__log.info(
                             "Download file: '%s' -> '%s'",
                             file_key,
@@ -208,9 +197,7 @@ class S3BotoDownloadWorker:
                         )
 
                         try:
-                            full_path.parent.mkdir(
-                                mode=0o770, parents=True, exist_ok=True
-                            )
+                            full_path.parent.mkdir(mode=0o770, parents=True, exist_ok=True)
                             self.download_file(str(full_path), file_key, file["Size"])
 
                             self.__log.info(f"Download complete for {str(full_path)}. ")
@@ -225,9 +212,7 @@ class S3BotoDownloadWorker:
                             progress_logger.set_state(
                                 full_path,
                                 file_metadata,
-                                state=DownloadState(
-                                    download_successful=False, errors=[str(e)]
-                                ),
+                                state=DownloadState(download_successful=False, errors=[str(e)]),
                             )
 
                             raise e

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/fastq_validation.py

@@ -29,9 +29,7 @@ def is_gzipped(file_path: str | PathLike) -> bool:
 
 
 @contextmanager
-def open_fastq(
-    file_path: str | PathLike, progress=True
-) -> Generator[TextIO, None, None]:
+def open_fastq(file_path: str | PathLike, progress=True) -> Generator[TextIO, None, None]:
     """
     Open a FASTQ file, handling both regular and gzipped formats.
 
@@ -103,9 +101,7 @@ def validate_fastq_file(fastq_file: str | PathLike) -> tuple[int, set[int], list
 
     # Check if the number of lines in a FASTQ file is a multiple of 4.
     if num_lines % 4 != 0:
-        errors.append(
-            f"{fastq_file}: Number of lines is not a multiple of 4! Found {num_lines} lines."
-        )
+        errors.append(f"{fastq_file}: Number of lines is not a multiple of 4! Found {num_lines} lines.")
     else:
         log.debug("%s: %s lines", fastq_file, num_lines)
 
@@ -132,9 +128,7 @@ def validate_single_end_reads(fastq_file: str | PathLike) -> Generator[str]:
     yield from errors
 
 
-def validate_paired_end_reads(
-    fastq_file1: str | PathLike, fastq_file2: str | PathLike
-) -> Generator[str]:
+def validate_paired_end_reads(fastq_file1: str | PathLike, fastq_file2: str | PathLike) -> Generator[str]:
     """
     Validate two paired-end FASTQ files.
 

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/file_operations.py

@@ -69,9 +69,7 @@ def calculate_md5(file_path, chunk_size=2**16, progress=True) -> str:
     md5_hash = hashlib.md5()  # noqa: S324
     with open(file_path, "rb") as f:
         if progress and (total_size > chunk_size):
-            with tqdm(
-                total=total_size, unit="B", unit_scale=True, desc="Calculating MD5"
-            ) as pbar:
+            with tqdm(total=total_size, unit="B", unit_scale=True, desc="Calculating MD5") as pbar:
                 while chunk := f.read(chunk_size):
                     md5_hash.update(chunk)
                     pbar.update(len(chunk))
@@ -117,9 +115,7 @@ def read_multiple_json(input: TextIO, buffer_size=65536, max_buffer_size=1342177
         raise ValueError("Remaining data is not empty. Is there invalid JSON?")
 
 
-def is_relative_subdirectory(
-    relative_path: str | PathLike, root_directory: str | PathLike
-) -> bool:
+def is_relative_subdirectory(relative_path: str | PathLike, root_directory: str | PathLike) -> bool:
     """
     Check if the target path is a subdirectory of the root path
     using os.path.commonpath() without checking the file system.
@@ -223,9 +219,7 @@ class Crypt4GH:
         if passphrase:
             passphrase_callback = lambda: passphrase
         else:
-            passphrase_callback = partial(
-                getpass, prompt=f"Passphrase for {seckey_path}: "
-            )
+            passphrase_callback = partial(getpass, prompt=f"Passphrase for {seckey_path}: ")
 
         return crypt4gh.keys.get_private_key(seckeypath, passphrase_callback)
 
@@ -254,9 +248,7 @@ class Crypt4GH:
             ) as pbar_in_fd,
         ):
             crypt4gh.lib.decrypt(
-                keys=[
-                    (0, private_key, None)
-                ],  # list of (method, privkey, recipient_pubkey=None),
+                keys=[(0, private_key, None)],  # list of (method, privkey, recipient_pubkey=None),
                 infile=pbar_in_fd,
                 outfile=out_fd,
             )

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/models/config.py

@@ -97,22 +97,14 @@ class ConfigModel(StrictBaseModel):
     @field_validator("grz_public_key")
     @classmethod
     def check_grz_public_key(cls, v):
-        if (
-            v is not None
-            and "BEGIN CRYPT4GH PUBLIC KEY" not in v
-            and "END CRYPT4GH PUBLIC KEY" not in v
-        ):
+        if v is not None and "BEGIN CRYPT4GH PUBLIC KEY" not in v and "END CRYPT4GH PUBLIC KEY" not in v:
             raise ValueError("Invalid public key format")
         return v
 
     @model_validator(mode="after")
     def validate_grz_public_key(self) -> Self:
         if self.grz_public_key is None and self.grz_public_key_path is None:
-            raise ValueError(
-                "Either grz_public_key or grz_public_key_path must be set."
-            )
+            raise ValueError("Either grz_public_key or grz_public_key_path must be set.")
         if self.grz_public_key is not None and self.grz_public_key_path is not None:
-            raise ValueError(
-                "Only one of grz_public_key or grz_public_key_path must be set."
-            )
+            raise ValueError("Only one of grz_public_key or grz_public_key_path must be set.")
         return self

{grz_cli-0.1.2 → grz_cli-0.1.3}/src/grz_cli/models/v1_0_0/metadata.py

@@ -16,12 +16,14 @@
 
 from __future__ import annotations
 
+import json
 import logging
 from collections.abc import Generator
 from datetime import date
 from enum import StrEnum
+from importlib.resources import files
 from pathlib import Path
-from typing import Annotated, Any
+from typing import Annotated, Any, Self
 
 from pydantic import (
     BaseModel,
@@ -105,17 +107,15 @@ class Submission(StrictBaseModel):
 
     genomic_data_center_id: Annotated[
         str,
-        StringConstraints(pattern=r"^(GRZ|KDK)[A-Z0-9]{3}[0-9]{3}$"),
+        StringConstraints(pattern=r"^(GRZ)[A-Z0-9]{3}[0-9]{3}$"),
     ]
     """
     ID of the genomic data center in the format GRZXXXnnn.
     """
 
-    clinical_data_node_id: Annotated[
-        str, StringConstraints(pattern=r"^(GRZ|KDK)[A-Z0-9]{3}[0-9]{3}$")
-    ]
+    clinical_data_node_id: Annotated[str, StringConstraints(pattern=r"^(KDK)[A-Z0-9]{3}[0-9]{3}$")]
     """
-    ID of the genomic data center in the format KDKXXXnnn.
+    ID of the clinical data node in the format KDKXXXnnn.
     """
 
     genomic_study_type: GenomicStudyType
@@ -262,10 +262,13 @@ class LibraryType(StrEnum):
     """
 
     panel = "panel"
+    panel_lr = "panel_lr"
     wes = "wes"
+    wes_lr = "wes_lr"
     wgs = "wgs"
     wgs_lr = "wgs_lr"
     wxs = "wxs"
+    wxs_lr = "wxs_lr"
     other = "other"
     unknown = "unknown"
 
@@ -433,6 +436,16 @@ class File(StrictBaseModel):
         return self.file_path + ".c4gh"
 
 
+class PercentBasesAboveQualityThreshold(StrictBaseModel):
+    """Percentage of bases with a specified minimum quality threshold"""
+
+    minimum_quality: Annotated[float, Field(strict=True, ge=0.0)]
+    """The minimum quality score threshold"""
+
+    percent: Annotated[float, Field(strict=True, ge=0.0, le=100.0)]
+    """Percentage of bases that meet or exceed the minimum quality score"""
+
+
 class SequenceData(StrictBaseModel):
     bioinformatics_pipeline_name: str
     """
@@ -449,9 +462,9 @@ class SequenceData(StrictBaseModel):
     Reference genome used
     """
 
-    percent_bases_q30: Annotated[float, Field(strict=True, ge=0.0, le=100.0)]
+    percent_bases_above_quality_threshold: PercentBasesAboveQualityThreshold
     """
-    Percentage of bases with Q30 or higher
+    Percentage of bases with a specified minimum quality threshold
     """
 
     mean_depth_of_coverage: Annotated[float, Field(strict=True, ge=0.0)]
@@ -464,9 +477,7 @@ class SequenceData(StrictBaseModel):
     Minimum coverage
     """
 
-    targeted_regions_above_min_coverage: Annotated[
-        float, Field(strict=True, ge=0.0, le=1.0)
-    ]
+    targeted_regions_above_min_coverage: Annotated[float, Field(strict=True, ge=0.0, le=1.0)]
     """
     Fraction of targeted regions that are above minimum coverage
     """
@@ -590,9 +601,7 @@ class LabDatum(StrictBaseModel):
     The sequencing layout, aka the end type of sequencing.
     """
 
-    tumor_cell_count: Annotated[
-        float | None, Field(alias="tumorCellCount", ge=0.0, le=100.0)
-    ] = None
+    tumor_cell_count: Annotated[float | None, Field(alias="tumorCellCount", ge=0.0, le=100.0)] = None
     """
     Tumor cell count in %
     """
@@ -607,6 +616,15 @@ class LabDatum(StrictBaseModel):
     Sequence data generated from the wet lab experiment.
     """
 
+    @model_validator(mode="after")
+    def validate_sequencing_setup(self) -> Self:
+        if self.library_type in {LibraryType.wxs, LibraryType.wxs_lr} and self.sequence_type != SequenceType.rna:
+            raise ValueError(
+                f"Error in lab datum '{self.lab_data_name}': "
+                f"WXS requires RNA sequencing, but got '{self.sequence_type}'."
+            )
+        return self
+
 
 class Donor(StrictBaseModel):
     case_id: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{32}$")]
@@ -698,12 +716,17 @@ class Donor(StrictBaseModel):
         def contains_bed_files(sequence_data: SequenceData) -> bool:
             return any(f.file_type == FileType.bed for f in sequence_data.files)
 
-        lib_type = {LibraryType.panel, LibraryType.wes}
+        lib_types = {
+            LibraryType.panel,
+            LibraryType.wes,
+            LibraryType.wxs,
+            LibraryType.panel_lr,
+            LibraryType.wes_lr,
+            LibraryType.wxs_lr,
+        }
 
         for lab_datum in self.lab_data:
-            if lab_datum.library_type in lib_type and not contains_bed_files(
-                lab_datum.sequence_data
-            ):
+            if lab_datum.library_type in lib_types and not contains_bed_files(lab_datum.sequence_data):
                 raise ValueError(
                     f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.case_id}'."
                 )
@@ -751,12 +774,8 @@ class Donor(StrictBaseModel):
                         )
 
                 # check if there is an equal number of R1 and R2 files
-                r1_fastq_files = [
-                    i for i in fastq_files if i.read_order == ReadOrder.r1
-                ]
-                r2_fastq_files = [
-                    i for i in fastq_files if i.read_order == ReadOrder.r2
-                ]
+                r1_fastq_files = [i for i in fastq_files if i.read_order == ReadOrder.r1]
+                r2_fastq_files = [i for i in fastq_files if i.read_order == ReadOrder.r2]
 
                 if len(r1_fastq_files) != len(r2_fastq_files):
                     raise ValueError(
@@ -790,60 +809,92 @@ class GrzSubmissionMetadata(StrictBaseModel):
             case GenomicStudyType.single:
                 # Check if the submission has at least one donor
                 if not self.donors:
-                    raise ValueError(
-                        "At least one donor is required for a single study."
-                    )
+                    raise ValueError("At least one donor is required for a single study.")
             case GenomicStudyType.duo:
                 # Check if the submission has at least two donors
                 if len(self.donors) < 2:
-                    raise ValueError(
-                        "At least two donors are required for a duo study."
-                    )
+                    raise ValueError("At least two donors are required for a duo study.")
             case GenomicStudyType.trio:
                 # Check if the submission has at least three donors
                 if len(self.donors) < 3:
-                    raise ValueError(
-                        "At least three donors are required for a trio study."
-                    )
+                    raise ValueError("At least three donors are required for a trio study.")
 
         return self
 
     @model_validator(mode="after")
-    def validate_minimum_mean_coverage(self):
+    def validate_thresholds(self):
         """
         Check if the submission meets the minimum mean coverage requirements.
         """
-        thresholds_somatic = {
-            (LibraryType.panel, SequenceSubtype.somatic): 500,
-            (LibraryType.wes, SequenceSubtype.somatic): 250,
-            (LibraryType.wes, SequenceSubtype.germline): 100,
-            (LibraryType.wgs, SequenceSubtype.somatic): 80,
-            (LibraryType.wgs, SequenceSubtype.germline): 40,
-        }
-        thresholds_rare_disease = {(LibraryType.wgs, SequenceSubtype.germline): 30}
-
-        study_subtype = self.submission.genomic_study_subtype
-        thresholds = (
-            thresholds_rare_disease
-            if study_subtype == GenomicStudySubtype.germline_only
-            else thresholds_somatic
-        )
+        threshold_definitions = _load_thresholds()
 
         for donor in self.donors:
             for lab_datum in donor.lab_data:
-                threshold = thresholds.get(
-                    (lab_datum.library_type, lab_datum.sequence_subtype)
+                key = (
+                    self.submission.genomic_study_subtype,
+                    lab_datum.library_type,
+                    lab_datum.sequence_subtype,
                 )
-                if threshold is None:
-                    log.warning(
-                        f"Threshold for {lab_datum.library_type} - {lab_datum.sequence_subtype} not found!"
-                    )
+                thresholds = threshold_definitions.get(key)
+                if thresholds is None:
+                    log.warning(f"Thresholds for {key} not found! Skipping.")
+                    continue
 
-                sequence_data = lab_datum.sequence_data
-                if sequence_data.mean_depth_of_coverage < threshold:
-                    raise ValueError(
-                        f"Mean depth of coverage for donor '{donor.case_id}', lab datum '{lab_datum.lab_data_name}' "
-                        f"below threshold ({threshold}): {sequence_data.mean_depth_of_coverage}"
-                    )
+                _check_thresholds(donor, lab_datum, thresholds)
 
         return self
+
+
+def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, Any]):
+    sequence_data = lab_datum.sequence_data
+    case_id = donor.case_id
+    lab_data_name = lab_datum.lab_data_name
+
+    mean_depth_of_coverage_t = thresholds.get("meanDepthOfCoverage")
+    mean_depth_of_coverage_v = sequence_data.mean_depth_of_coverage
+    if mean_depth_of_coverage_t and mean_depth_of_coverage_v < mean_depth_of_coverage_t:
+        raise ValueError(
+            f"Mean depth of coverage for donor '{case_id}', lab datum '{lab_data_name}' "
+            f"below threshold: {mean_depth_of_coverage_v} < {mean_depth_of_coverage_t}"
+        )
+
+    read_length_t = thresholds.get("readLength")
+    read_length_v = sequence_data.read_length
+    if read_length_t and read_length_v < read_length_t:
+        raise ValueError(
+            f"Read length for donor '{case_id}', lab datum '{lab_data_name}' "
+            f"below threshold: {read_length_v} < {read_length_t}"
+        )
+
+    if t := thresholds.get("targetedRegionsAboveMinCoverage"):
+        min_coverage_t = t.get("minCoverage")
+        min_coverage_v = sequence_data.min_coverage
+
+        fraction_above_t = t.get("fractionAbove")
+        fraction_above_v = sequence_data.targeted_regions_above_min_coverage
+
+        if min_coverage_t and min_coverage_v < min_coverage_t:
+            raise ValueError(
+                f"Minimum coverage for donor '{case_id}', lab datum '{lab_data_name}' "
+                f"below threshold: {min_coverage_v} < {min_coverage_t}"
+            )
+        if fraction_above_t and fraction_above_v < fraction_above_t:
+            raise ValueError(
+                f"Fraction of targeted regions above minimum coverage for donor '{case_id}', "
+                f"lab datum '{lab_data_name}' below threshold: "
+                f"{fraction_above_v} < {fraction_above_t}"
+            )
+
+
+type Thresholds = dict[tuple[str, str, str], dict[str, Any]]
+
+
+def _load_thresholds() -> Thresholds:
+    threshold_definitions = json.load(
+        files("grz_cli").joinpath("resources", "thresholds.json").open("r", encoding="utf-8")
+    )
+    threshold_definitions = {
+        (d["genomicStudySubtype"], d["libraryType"], d["sequenceSubtype"]): d["thresholds"]
+        for d in threshold_definitions
+    }
+    return threshold_definitions