grz-cli 0.1.3__tar.gz → 0.1.4__tar.gz

{grz_cli-0.1.3/src/grz_cli.egg-info → grz_cli-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: grz-cli
-Version: 0.1.3
+Version: 0.1.4
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License: MIT
@@ -11,7 +11,6 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: boto3<2,>=1.35.39
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
@@ -26,7 +25,7 @@ Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
 Requires-Dist: pytest-mock<4,>=3.14.0; extra == "test"
 Requires-Dist: moto[s3]<6,>=5.0.16; extra == "test"
 Requires-Dist: pytest-sugar<2,>=1.0.0; extra == "test"
-Requires-Dist: ruff<1,>=0.8.1; extra == "test"
+Requires-Dist: ruff<1,>=0.9.6; extra == "test"
 Requires-Dist: numpy<3,>=2.1.2; extra == "test"
 
 # GRZ CLI

{grz_cli-0.1.3 → grz_cli-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "grz-cli"
-version = "0.1.3"
+version = "0.1.4"
 description = "Tool for validation, encryption and upload of MV submissions to GDCs."
 readme = "README.md"
 license = { text = "MIT" }
@@ -46,7 +46,7 @@ test = [
     "pytest-mock >=3.14.0,<4",
     "moto[s3] >=5.0.16,<6",
     "pytest-sugar >=1.0.0,<2",
-    "ruff >=0.8.1,<1",
+    "ruff >=0.9.6,<1",
     "numpy >=2.1.2,<3"
 ]
 
@@ -154,21 +154,25 @@ allowlist_externals = ["pytest", "ruff"]
 isolated_build = true
 
 [tool.tox.env_run_base]
+runner = "uv-venv-lock-runner"
 description = "run unit tests"
 dependency_groups = ["test"]
 commands = [["pytest", { replace = "posargs", default = ["tests"], extend = true }]]
 
 [tool.tox.env."format-check"]
+runner = "uv-venv-lock-runner"
 skip_install = true
 dependency_groups = ["lint"]
 commands = [["ruff", "format", "--check"]]
 
 [tool.tox.env."lints"]
+runner = "uv-venv-lock-runner"
 skip_install = true
 dependency_groups = ["lint"]
 commands = [["ruff", "check"]]
 
 [tool.tox.env."mypy"]
+runner = "uv-venv-lock-runner"
 skip_install = false
 dependency_groups = ["lint"]
 commands = [["mypy"]]
@@ -177,8 +181,9 @@ commands = [["mypy"]]
 dev-dependencies = [
     "datamodel-code-generator>=0.26.2",
     "mypy>=1.13.0",
-    "ruff>=0.8.1",
+    "ruff>=0.9.6",
     "tox>=4.23.2",
+    "tox-uv>=1.20.1",
     "types-pyyaml>=6.0.12.20240917",
     "types-tqdm>=4.66.0.20240417",
 ]
@@ -192,3 +197,6 @@ packages = "src"
 addopts = [
     "--import-mode=importlib",
 ]
+
+[tool.setuptools]
+license-files = []

{grz_cli-0.1.3 → grz_cli-0.1.4}/src/grz_cli/download.py

@@ -146,7 +146,7 @@ class S3BotoDownloadWorker:
             if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
             else MULTIPART_CHUNKSIZE
         )
-        self.__log.debug(f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize/chunksize} chunks")
+        self.__log.debug(f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize / chunksize} chunks")
 
         config = TransferConfig(
             multipart_threshold=MULTIPART_THRESHOLD,

{grz_cli-0.1.3 → grz_cli-0.1.4}/src/grz_cli/models/v1_0_0/metadata.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 
 import json
 import logging
+import typing
 from collections.abc import Generator
 from datetime import date
 from enum import StrEnum
@@ -90,9 +91,9 @@ class Submission(StrictBaseModel):
     The options are: 'initial' for first submission, 'followup' is for followup submissions, 'addition' for additional submission, 'correction' for correction
     """
 
-    tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{32}$")]
+    tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")]
     """
-    The VNg of the genomic data of the patient that will be reimbursed --> a unique 32-byte hex ID.
+    The VNg of the genomic data of the patient that will be reimbursed --> a unique 32-length byte code represented in a hex string of length 64.
     """
 
     local_case_id: str | None = None
@@ -134,9 +135,9 @@ class Submission(StrictBaseModel):
     """
 
 
-class Sex(StrEnum):
+class Gender(StrEnum):
     """
-    Sex of the donor.
+    Gender of the donor.
     """
 
     male = "male"
@@ -156,6 +157,7 @@ class Relation(StrEnum):
     sister = "sister"
     child = "child"
     self = "self"
+    other = "other"
 
 
 class MvConsentScope(StrictBaseModel):
@@ -284,6 +286,7 @@ class EnrichmentKitManufacturer(StrEnum):
     neb = "NEB"
     other = "other"
     unknown = "unknown"
+    none = "none"
 
 
 class SequencingLayout(StrEnum):
@@ -308,6 +311,22 @@ class TumorCellCountMethod(StrEnum):
     unknown = "unknown"
 
 
+class TumorCellCount(StrictBaseModel):
+    """
+    Tuple of tumor cell counts and how they were determined.
+    """
+
+    count: Annotated[float, Field(ge=0.0, le=100.0)]
+    """
+    Tumor cell count in %
+    """
+
+    method: TumorCellCountMethod
+    """
+    Method used to determine cell count.
+    """
+
+
 class CallerUsedItem(StrictBaseModel):
     name: str
     """
@@ -443,7 +462,9 @@ class PercentBasesAboveQualityThreshold(StrictBaseModel):
     """The minimum quality score threshold"""
 
     percent: Annotated[float, Field(strict=True, ge=0.0, le=100.0)]
-    """Percentage of bases that meet or exceed the minimum quality score"""
+    """
+    Percentage of bases with a specified minimum quality threshold, according to https://www.bfarm.de/SharedDocs/Downloads/DE/Forschung/modellvorhaben-genomsequenzierung/Qs-durch-GRZ.pdf?__blob=publicationFile
+    """
 
 
 class SequenceData(StrictBaseModel):
@@ -502,6 +523,12 @@ class SequenceData(StrictBaseModel):
     List of files generated and required in this analysis.
     """
 
+    def contains_files(self, file_type: FileType) -> bool:
+        return any(f.file_type == file_type for f in self.files)
+
+    def list_files(self, file_type: FileType) -> list[File]:
+        return [f for f in self.files if f.file_type == file_type]
+
 
 class LabDatum(StrictBaseModel):
     lab_data_name: str
@@ -601,21 +628,19 @@ class LabDatum(StrictBaseModel):
     The sequencing layout, aka the end type of sequencing.
     """
 
-    tumor_cell_count: Annotated[float | None, Field(alias="tumorCellCount", ge=0.0, le=100.0)] = None
-    """
-    Tumor cell count in %
-    """
-
-    tumor_cell_count_method: TumorCellCountMethod | None = None
+    tumor_cell_count: list[TumorCellCount] | None = None
     """
-    Method used to determine cell count.
+    Tuple of tumor cell counts and how they were determined.
     """
 
-    sequence_data: SequenceData
+    sequence_data: SequenceData | None = None
     """
     Sequence data generated from the wet lab experiment.
     """
 
+    def has_sequence_data(self) -> bool:
+        return self.sequence_data is not None
+
     @model_validator(mode="after")
     def validate_sequencing_setup(self) -> Self:
         if self.library_type in {LibraryType.wxs, LibraryType.wxs_lr} and self.sequence_type != SequenceType.rna:
@@ -627,14 +652,19 @@ class LabDatum(StrictBaseModel):
 
 
 class Donor(StrictBaseModel):
-    case_id: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{32}$")]
+    tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")]
     """
-    The VNg of the genomic data of the donor --> a unique 32-byte hex ID.
+    The VNg of the genomic data of the donor --> a unique 32-length byte code represented in a hex string of length 64.
     """
 
-    sex: Sex
+    local_case_id: str | None = None
     """
-    Sex of the donor.
+    A local case identifier for synchronizing locally
+    """
+
+    gender: Gender
+    """
+    Gender of the donor.
     """
 
     relation: Relation
@@ -707,15 +737,21 @@ class Donor(StrictBaseModel):
     Lab data related to the donor.
     """
 
+    @model_validator(mode="after")
+    def warn_empty_sequence_data(self):
+        for lab_datum in self.lab_data:
+            if not lab_datum.has_sequence_data():
+                log.warning(
+                    f"No sequence data found for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'. "
+                    "Is this a submission without sequence data?"
+                )
+        return self
+
     @model_validator(mode="after")
     def validate_target_bed_files_exist(self):
         """
         Check if the submission has the required bed files for panel sequencing.
         """
-
-        def contains_bed_files(sequence_data: SequenceData) -> bool:
-            return any(f.file_type == FileType.bed for f in sequence_data.files)
-
         lib_types = {
             LibraryType.panel,
             LibraryType.wes,
@@ -726,41 +762,36 @@ class Donor(StrictBaseModel):
         }
 
         for lab_datum in self.lab_data:
-            if lab_datum.library_type in lib_types and not contains_bed_files(lab_datum.sequence_data):
-                raise ValueError(
-                    f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.case_id}'."
-                )
+            if (
+                lab_datum.has_sequence_data()
+                and lab_datum.library_type in lib_types
+                and not lab_datum.sequence_data.contains_files(FileType.bed)
+            ):
+                raise ValueError(f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'.")
 
         return self
 
     @model_validator(mode="after")
     def validate_vcf_file_exists(self):
         """
-        check if there is a VCF file
+        Check if there is a VCF file
         """
-
-        def contains_vcf_files(sequence_data: SequenceData) -> bool:
-            return any(f.file_type == FileType.vcf for f in sequence_data.files)
-
         for lab_datum in self.lab_data:
-            if not contains_vcf_files(lab_datum.sequence_data):
-                raise ValueError(
-                    f"VCF file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.case_id}'."
-                )
+            if lab_datum.has_sequence_data() and not lab_datum.sequence_data.contains_files(FileType.vcf):
+                raise ValueError(f"VCF file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'.")
 
         return self
 
     @model_validator(mode="after")
     def validate_fastq_file_exists(self):
         """
-        check if there is a FASTQ file
+        Check if there is a FASTQ file
         """
-
-        def get_fastq_files(sequence_data: SequenceData):
-            return [f for f in sequence_data.files if f.file_type == FileType.fastq]
-
         for lab_datum in self.lab_data:
-            fastq_files = get_fastq_files(lab_datum.sequence_data)
+            if not lab_datum.has_sequence_data():
+                # Skip if no sequence data is present
+                continue
+            fastq_files = lab_datum.sequence_data.list_files(FileType.fastq)
 
             if len(fastq_files) == 0:
                 raise ValueError("No FASTQ file found!")
@@ -769,7 +800,7 @@ class Donor(StrictBaseModel):
                 for i in fastq_files:
                     if i.read_order is None:
                         raise ValueError(
-                            f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.case_id}': "
+                            f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.tan_g}': "
                             f"No read order specified for FASTQ file '{i.file_path}'!"
                         )
 
@@ -779,7 +810,7 @@ class Donor(StrictBaseModel):
 
                 if len(r1_fastq_files) != len(r2_fastq_files):
                     raise ValueError(
-                        f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.case_id}': "
+                        f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.tan_g}': "
                         f"Paired end sequencing layout but number of R1 FASTQ files ({len(r1_fastq_files)})"
                         f" differs from number of R2 FASTQ files ({len(r2_fastq_files)})!"
                     )
@@ -821,6 +852,37 @@ class GrzSubmissionMetadata(StrictBaseModel):
 
         return self
 
+    @model_validator(mode="after")
+    def check_for_tumor_cell_count(self):
+        """
+        Check if oncology samples have tumor cell counts.
+        """
+        for donor in self.donors:
+            case_id = donor.tan_g
+            for lab_datum in donor.lab_data:
+                if lab_datum.sequence_subtype == SequenceSubtype.somatic and lab_datum.tumor_cell_count is None:
+                    raise ValueError(
+                        f"Missing tumor cell count for donor '{case_id}', lab datum '{lab_datum.lab_data_name}'!"
+                    )
+
+        return self
+
+    @model_validator(mode="after")
+    def check_duplicate_lab_data_names(self):
+        """
+        Check if the submission contains lab data with the same name within one donor.
+        """
+        for donor in self.donors:
+            case_id = donor.tan_g
+            lab_data_names = set()
+            for lab_datum in donor.lab_data:
+                if lab_datum.lab_data_name in lab_data_names:
+                    raise ValueError(f"Duplicate lab datum '{lab_datum.lab_data_name}' in donor '{case_id}'")
+                else:
+                    lab_data_names.add(lab_datum.lab_data_name)
+
+        return self
+
     @model_validator(mode="after")
     def validate_thresholds(self):
         """
@@ -837,7 +899,20 @@ class GrzSubmissionMetadata(StrictBaseModel):
                 )
                 thresholds = threshold_definitions.get(key)
                 if thresholds is None:
-                    log.warning(f"Thresholds for {key} not found! Skipping.")
+                    allowed_combinations = sorted(list(threshold_definitions.keys()))
+                    allowed_combinations = "\n".join([f"  - {combination}" for combination in allowed_combinations])
+                    names = (
+                        "submission.genomicStudySubtype",
+                        "labData.libraryType",
+                        "labData.sequenceSubtype",
+                    )
+                    info = dict(zip(names, key, strict=True))
+                    log.warning(
+                        f"No thresholds for the specified combination {info} found (donor {donor.tan_g})!\n"
+                        f"Valid combinations:\n{allowed_combinations}.\n"
+                        f"See https://www.bfarm.de/SharedDocs/Downloads/DE/Forschung/modellvorhaben-genomsequenzierung/Qs-durch-GRZ.pdf?__blob=publicationFile for more details.\n"
+                        f"Skipping threshold validation."
+                    )
                     continue
 
                 _check_thresholds(donor, lab_datum, thresholds)
@@ -846,9 +921,13 @@ class GrzSubmissionMetadata(StrictBaseModel):
 
 
 def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, Any]):
-    sequence_data = lab_datum.sequence_data
-    case_id = donor.case_id
+    if not lab_datum.has_sequence_data():
+        # Skip if no sequence data is present; warning issues in the validator `warn_empty_sequence_data` of `Donor`.
+        return
+    case_id = donor.tan_g
     lab_data_name = lab_datum.lab_data_name
+    # mypy cannot reason about the `has_sequence_data` check
+    sequence_data = typing.cast(SequenceData, lab_datum.sequence_data)
 
     mean_depth_of_coverage_t = thresholds.get("meanDepthOfCoverage")
     mean_depth_of_coverage_v = sequence_data.mean_depth_of_coverage

{grz_cli-0.1.3 → grz_cli-0.1.4}/src/grz_cli/parser.py

@@ -9,6 +9,8 @@ from itertools import groupby
 from os import PathLike
 from pathlib import Path
 
+from pydantic import ValidationError
+
 from .download import S3BotoDownloadWorker
 from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
 from .file_operations import Crypt4GH, calculate_sha256
@@ -60,7 +62,11 @@ class SubmissionMetadata:
         try:
             with open(file_path, encoding="utf-8") as jsonfile:
                 metadata = json.load(jsonfile)
-                metadata_model = GrzSubmissionMetadata(**metadata)
+                try:
+                    metadata_model = GrzSubmissionMetadata(**metadata)
+                except ValidationError as ve:
+                    cls.__log.error("Invalid metadata format in metadata file: %s", file_path)
+                    raise SystemExit(ve) from ve
                 return metadata_model
         except json.JSONDecodeError as e:
             cls.__log.error("Invalid JSON format in metadata file: %s", file_path)