grz-cli 0.1.4__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {grz_cli-0.1.4/src/grz_cli.egg-info → grz_cli-0.2.0}/PKG-INFO +1 -1
  2. {grz_cli-0.1.4 → grz_cli-0.2.0}/pyproject.toml +1 -1
  3. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/cli.py +3 -2
  4. {grz_cli-0.1.4/src/grz_cli/models/v1_0_0 → grz_cli-0.2.0/src/grz_cli/models/v1_1_1}/metadata.py +226 -134
  5. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/parser.py +2 -2
  6. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/progress_logging.py +1 -1
  7. {grz_cli-0.1.4 → grz_cli-0.2.0/src/grz_cli.egg-info}/PKG-INFO +1 -1
  8. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/SOURCES.txt +2 -4
  9. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_file_progress_logging.py +1 -1
  10. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_parser.py +5 -1
  11. grz_cli-0.1.4/LICENSE +0 -21
  12. grz_cli-0.1.4/src/grz_cli/resources/grz-schema.json +0 -595
  13. {grz_cli-0.1.4 → grz_cli-0.2.0}/MANIFEST.in +0 -0
  14. {grz_cli-0.1.4 → grz_cli-0.2.0}/README.md +0 -0
  15. {grz_cli-0.1.4 → grz_cli-0.2.0}/setup.cfg +0 -0
  16. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/__init__.py +0 -0
  17. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/__init__.py +0 -0
  18. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/constants.py +0 -0
  19. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/download.py +0 -0
  20. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/fastq_validation.py +0 -0
  21. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/file_operations.py +0 -0
  22. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/logging_setup.py +0 -0
  23. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/models/__init__.py +0 -0
  24. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/models/config.py +0 -0
  25. {grz_cli-0.1.4/src/grz_cli/models/v1_0_0 → grz_cli-0.2.0/src/grz_cli/models/v1_1_1}/__init__.py +0 -0
  26. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/resources/thresholds.json +0 -0
  27. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/states.py +0 -0
  28. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/upload.py +0 -0
  29. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/dependency_links.txt +0 -0
  30. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/entry_points.txt +0 -0
  31. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/requires.txt +0 -0
  32. {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/top_level.txt +0 -0
  33. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_cli.py +0 -0
  34. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_download.py +0 -0
  35. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_fastq_validation.py +0 -0
  36. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_file_operations.py +0 -0
  37. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_read_multiple_json.py +0 -0
  38. {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_upload.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: grz-cli
3
- Version: 0.1.4
3
+ Version: 0.2.0
4
4
  Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
5
5
  Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
6
6
  License: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "grz-cli"
7
- version = "0.1.4"
7
+ version = "0.2.0"
8
8
  description = "Tool for validation, encryption and upload of MV submissions to GDCs."
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -2,6 +2,7 @@
2
2
  CLI module for handling command-line interface operations.
3
3
  """
4
4
 
5
+ import importlib
5
6
  import logging
6
7
  import logging.config
7
8
  import sys
@@ -107,7 +108,7 @@ class OrderedGroup(click.Group):
107
108
  cls=OrderedGroup,
108
109
  help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
109
110
  )
110
- @click.version_option(version="0.1", prog_name="grz-cli")
111
+ @click.version_option(version=importlib.metadata.version("grz-cli"), prog_name="grz-cli")
111
112
  @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
112
113
  @click.option(
113
114
  "--log-level",
@@ -229,7 +230,7 @@ def decrypt(
229
230
  log.error("GRZ private key path is required for decryption.")
230
231
  sys.exit(1)
231
232
 
232
- log.info("Starting encryption...")
233
+ log.info("Starting decryption...")
233
234
 
234
235
  submission_dir = Path(submission_dir)
235
236
 
@@ -1,19 +1,3 @@
1
- # generated by datamodel-codegen:
2
- # filename: grz-schema.json
3
- # timestamp: 2024-11-21T12:40:39+00:00
4
- # version: 0.26.2
5
- ### Command run:
6
- ## datamodel-codegen --output-model-type pydantic_v2.BaseModel \
7
- ## --input src/grz_cli/resources/grz-schema.json \
8
- ## --use-schema-description --use-field-description --snake-case-field --use-standard-collections --use-union-operator \
9
- ## --field-constraints --use-annotated --enable-version-header --reuse-model --use-subclass-enum --union-mode smart \
10
- ## --class-name 'GrzSubmissionMetadata' --use-double-quotes --target-python-version 3.12
11
- ### Custom modifications:
12
- ### - disallow extra fields (no additional properties allowed)
13
- ### - validate_assignment (i.e. when setting properties, re-trigger validations)
14
- ### - use_enum_values (use values [strings] instead of enum variants)
15
- ### - remove those aliases which are covered by a global `alias_generator=to_camel` setting
16
-
17
1
  from __future__ import annotations
18
2
 
19
3
  import json
@@ -23,6 +7,7 @@ from collections.abc import Generator
23
7
  from datetime import date
24
8
  from enum import StrEnum
25
9
  from importlib.resources import files
10
+ from itertools import groupby
26
11
  from pathlib import Path
27
12
  from typing import Annotated, Any, Self
28
13
 
@@ -37,6 +22,8 @@ from pydantic.alias_generators import to_camel
37
22
 
38
23
  from grz_cli.file_operations import calculate_sha256 # type: ignore
39
24
 
25
+ SCHEMA_URL = "https://raw.githubusercontent.com/BfArM-MVH/MVGenomseq/refs/tags/v1.1.1/GRZ/grz-schema.json"
26
+
40
27
  log = logging.getLogger(__name__)
41
28
 
42
29
 
@@ -80,6 +67,33 @@ class GenomicStudySubtype(StrEnum):
80
67
  germline_only = "germline-only"
81
68
 
82
69
 
70
+ class CoverageType(StrEnum):
71
+ """
72
+ Health insurance providers
73
+ """
74
+
75
+ GKV = "GKV"
76
+ PKV = "PKV"
77
+ BG = "BG"
78
+ SEL = "SEL"
79
+ SOZ = "SOZ"
80
+ GPV = "GPV"
81
+ PPV = "PPV"
82
+ BEI = "BEI"
83
+ SKT = "SKT"
84
+ UNK = "UNK"
85
+
86
+
87
+ class DiseaseType(StrEnum):
88
+ """
89
+ Type of the disease
90
+ """
91
+
92
+ oncological = "oncological"
93
+ rare = "rare"
94
+ hereditary = "hereditary"
95
+
96
+
83
97
  class Submission(StrictBaseModel):
84
98
  submission_date: date
85
99
  """
@@ -93,14 +107,19 @@ class Submission(StrictBaseModel):
93
107
 
94
108
  tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")]
95
109
  """
96
- The VNg of the genomic data of the patient that will be reimbursed --> a unique 32-length byte code represented in a hex string of length 64.
110
+ The VNg of the genomic data of the index patient that will be reimbursed --> a unique 32-length byte code represented in a hex string of length 64.
97
111
  """
98
112
 
99
- local_case_id: str | None = None
113
+ local_case_id: str
100
114
  """
101
115
  A local case identifier for synchronizing locally
102
116
  """
103
117
 
118
+ coverage_type: CoverageType
119
+ """
120
+ Health insurance provider
121
+ """
122
+
104
123
  submitter_id: Annotated[str, StringConstraints(pattern=r"^[0-9]{9}$")]
105
124
  """
106
125
  Institutional ID of the submitter according to §293 SGB V.
@@ -119,6 +138,11 @@ class Submission(StrictBaseModel):
119
138
  ID of the clinical data node in the format KDKXXXnnn.
120
139
  """
121
140
 
141
+ disease_type: DiseaseType
142
+ """
143
+ Type of the disease
144
+ """
145
+
122
146
  genomic_study_type: GenomicStudyType
123
147
  """
124
148
  whether additional persons are tested as well
@@ -156,49 +180,103 @@ class Relation(StrEnum):
156
180
  brother = "brother"
157
181
  sister = "sister"
158
182
  child = "child"
159
- self = "self"
183
+ index_ = "index"
160
184
  other = "other"
161
185
 
162
186
 
187
+ class MvConsentScopeType(StrEnum):
188
+ """
189
+ Consent or refusal to participate and consent, must be indicated for each option listed in the scope of consent.
190
+ """
191
+
192
+ permit = "permit"
193
+ deny = "deny"
194
+
195
+
196
+ class MvConsentScopeDomain(StrEnum):
197
+ """
198
+ Scope of consent or revocation.
199
+ """
200
+
201
+ mv_sequencing = "mvSequencing"
202
+ re_identification = "reIdentification"
203
+ case_identification = "caseIdentification"
204
+
205
+
163
206
  class MvConsentScope(StrictBaseModel):
164
207
  """
165
208
  The scope of the Modellvorhaben consent given by the donor.
166
209
  """
167
210
 
168
- consent_mv_sequencing: bool
211
+ type_: Annotated[MvConsentScopeType, Field(alias="type")]
212
+ """
213
+ Consent or refusal to participate and consent, must be indicated for each option listed in the scope of consent.
214
+ """
215
+
216
+ date: date
217
+ """
218
+ Date of signature of the pilot projects consent; in ISO 8601 format YYYY-MM-DD.
219
+ """
220
+
221
+ domain: MvConsentScopeDomain
222
+ """
223
+ Scope of consent or revocation.
224
+ """
225
+
226
+
227
+ class MvConsent(StrictBaseModel):
228
+ presentation_date: date | None = None
169
229
  """
170
- The donor's consent to participate in the Modellvorhaben and sequencing.
230
+ Date of delivery. Date (in ISO 8601 format YYYY-MM-DD) on which the Model Project Declaration of Participation
231
+ was presented to the patient, unless identical to the date of signature
171
232
  """
172
233
 
173
- consent_case_identification: bool
234
+ version: str
174
235
  """
175
- The donor's consent to identify the case.
236
+ Version of the declaration of participation.
237
+ Name and version of the declaration of participation in the MV GenomSeq, e.g.:
238
+ 'Patient Info TE Consent MVGenomSeq vers01'
176
239
  """
177
240
 
178
- consent_re_identification: bool
241
+ scope: list[MvConsentScope]
179
242
  """
180
- The donor's consent to be re-identified and to be re-contacted in case of new findings.
243
+ Modules of the consent to MV: must have at least a permit of mvSequencing
181
244
  """
182
245
 
246
+ @model_validator(mode="after")
247
+ def ensure_mv_sequencing_scope_is_present(self):
248
+ if not any(
249
+ scope.domain == MvConsentScopeDomain.mv_sequencing and scope.type_ == MvConsentScopeType.permit
250
+ for scope in self.scope
251
+ ):
252
+ raise ValueError("Must have at least a permit of mvSequencing")
253
+ return self
254
+
255
+
256
+ class ResearchConsentSchemaVersion(StrEnum):
257
+ v_2025_0_1 = "2025.0.1"
258
+
183
259
 
184
- class MvConsentTerminationScope(StrictBaseModel):
260
+ class ResearchConsent(StrictBaseModel):
185
261
  """
186
- The scope of the termination of the Modellvorhaben consent.
262
+ Research consents. Multiple declarations of consent are possible! Must be assigned to the respective data sets.
187
263
  """
188
264
 
189
- consent_mv_sequencing_termination: bool | None = None
265
+ schema_version: ResearchConsentSchemaVersion
190
266
  """
191
- The patient's termination of the Modellvorhaben consent to participate in the Modellvorhaben and sequencing
267
+ Schema version of de.medizininformatikinitiative.kerndatensatz.consent
192
268
  """
193
269
 
194
- consent_case_identification_termination: bool | None = None
270
+ presentation_date: date | None = None
195
271
  """
196
- The patient's termination of the Modellvorhaben consent to identify the case
272
+ Date of the delivery of the research consent in ISO 8601 format (YYYY-MM-DD)
197
273
  """
198
274
 
199
- consent_re_identification_termination: bool | None = None
275
+ scope: object
200
276
  """
201
- The patient's termination of the Modellvorhaben consent to re-identify the case and to re-contact the patient in case of new findings
277
+ Scope of the research consent in JSON format following the MII IG Consent v2025 FHIR schema.
278
+ See 'https://www.medizininformatik-initiative.de/Kerndatensatz/KDS_Consent_V2025/MII-IG-Modul-Consent.html' and
279
+ 'https://packages2.fhir.org/packages/de.medizininformatikinitiative.kerndatensatz.consent'.
202
280
  """
203
281
 
204
282
 
@@ -305,8 +383,8 @@ class TumorCellCountMethod(StrEnum):
305
383
  Method used to determine cell count.
306
384
  """
307
385
 
308
- pathology = "Pathology"
309
- bioinformatics = "Bioinformatics"
386
+ pathology = "pathology"
387
+ bioinformatics = "bioinformatics"
310
388
  other = "other"
311
389
  unknown = "unknown"
312
390
 
@@ -378,6 +456,11 @@ class File(StrictBaseModel):
378
456
  Type of the file; if BED file is submitted, only 1 file is allowed.
379
457
  """
380
458
 
459
+ read_length: Annotated[int, Field(strict=True, ge=0)] | None = None
460
+ """
461
+ The read length; in the case of long-read sequencing it is the rounded average read length.
462
+ """
463
+
381
464
  checksum_type: ChecksumType | None = ChecksumType.sha256
382
465
  """
383
466
  Type of checksum algorithm used
@@ -408,6 +491,12 @@ class File(StrictBaseModel):
408
491
  Indicates the lane
409
492
  """
410
493
 
494
+ @model_validator(mode="after")
495
+ def ensure_read_length_is_present_for_bam_and_fastq(self):
496
+ if self.file_type in {FileType.bam, FileType.fastq} and self.read_length is None:
497
+ raise ValueError(f"Read length missing for file '{self.file_path}' of type '{self.file_type}'.")
498
+ return self
499
+
411
500
  def validate_data(self, local_file_path: Path) -> Generator[str]:
412
501
  """
413
502
  Validates whether the provided file matches this metadata.
@@ -467,6 +556,15 @@ class PercentBasesAboveQualityThreshold(StrictBaseModel):
467
556
  """
468
557
 
469
558
 
559
+ class ReferenceGenome(StrEnum):
560
+ """
561
+ Reference genome used according to the Genome Reference Consortium (https://www.ncbi.nlm.nih.gov/grc)
562
+ """
563
+
564
+ GRCh37 = "GRCh37"
565
+ GRCh38 = "GRCh38"
566
+
567
+
470
568
  class SequenceData(StrictBaseModel):
471
569
  bioinformatics_pipeline_name: str
472
570
  """
@@ -478,9 +576,9 @@ class SequenceData(StrictBaseModel):
478
576
  Version or commit hash of the bioinformatics pipeline
479
577
  """
480
578
 
481
- reference_genome: str
579
+ reference_genome: ReferenceGenome
482
580
  """
483
- Reference genome used
581
+ Reference genome used according to the Genome Reference Consortium (https://www.ncbi.nlm.nih.gov/grc)
484
582
  """
485
583
 
486
584
  percent_bases_above_quality_threshold: PercentBasesAboveQualityThreshold
@@ -503,11 +601,6 @@ class SequenceData(StrictBaseModel):
503
601
  Fraction of targeted regions that are above minimum coverage
504
602
  """
505
603
 
506
- read_length: Annotated[int, Field(strict=True, ge=0)]
507
- """
508
- The read length; in the case of long-read sequencing it is the rounded average read length.
509
- """
510
-
511
604
  non_coding_variants: bool
512
605
  """
513
606
  The analysis includes non-coding variants -> true or false
@@ -652,14 +745,12 @@ class LabDatum(StrictBaseModel):
652
745
 
653
746
 
654
747
  class Donor(StrictBaseModel):
655
- tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")]
656
- """
657
- The VNg of the genomic data of the donor --> a unique 32-length byte code represented in a hex string of length 64.
658
- """
659
-
660
- local_case_id: str | None = None
748
+ donor_pseudonym: str
661
749
  """
662
- A local case identifier for synchronizing locally
750
+ A unique identifier given by the Leistungserbringer for each donor of a single, duo or trio sequencing;
751
+ the donorPseudonym needs to be identifiable by the Leistungserbringer
752
+ in case of changes to the consents by one of the donors.
753
+ For Index patient use TanG.
663
754
  """
664
755
 
665
756
  gender: Gender
@@ -672,64 +763,11 @@ class Donor(StrictBaseModel):
672
763
  Relationship of the donor in respect to the index patient, e.g. 'index', 'brother', 'mother', etc.
673
764
  """
674
765
 
675
- mv_consent_date: date
676
- """
677
- Date of the signature of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
678
- """
679
-
680
- mv_consent_presented_date: date | None = None
681
- """
682
- Date of the presentation of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
683
- """
684
-
685
- mv_consent_version: str
686
- """
687
- Version of the research consent
688
- """
689
-
690
- mv_consent_scope: MvConsentScope
691
- """
692
- The scope of the Modellvorhaben consent given by the donor.
693
- """
694
-
695
- mv_consent_termination_date: date | None = None
696
- """
697
- Date of the termination of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
698
- """
699
-
700
- mv_consent_termination_scope: MvConsentTerminationScope | None = None
701
- """
702
- The scope of the termination of the Modellvorhaben consent.
703
- """
704
-
705
- research_consent_date: date | None = None
706
- """
707
- Date of the signature of the research consent; Date in ISO 8601 format YYYY-MM-DD
708
- """
709
-
710
- research_consent_presented_date: date | None = None
711
- """
712
- Date of the presentation of the research consent; Date in ISO 8601 format YYYY-MM-DD
713
- """
714
-
715
- research_consent_version: str | None = None
716
- """
717
- Version of the research consent
718
- """
719
-
720
- research_consent_scope: dict[str, Any] | None = None
721
- """
722
- Scope of the research consent given by the patient; shall be given in JSON format.
723
- """
724
-
725
- research_consent_revocation_date: date | None = None
726
- """
727
- Date of the revocation of the research consent; Date in ISO 8601 format YYYY-MM-DD
728
- """
766
+ mv_consent: MvConsent
729
767
 
730
- research_consent_revocation_scope: dict[str, Any] | None = None
768
+ research_consents: list[ResearchConsent]
731
769
  """
732
- Scope of the revocation of the research consent given by the patient; shall be given in JSON format
770
+ Research consents. Multiple declarations of consent are possible! Must be assigned to the respective data sets.
733
771
  """
734
772
 
735
773
  lab_data: list[LabDatum]
@@ -742,7 +780,7 @@ class Donor(StrictBaseModel):
742
780
  for lab_datum in self.lab_data:
743
781
  if not lab_datum.has_sequence_data():
744
782
  log.warning(
745
- f"No sequence data found for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'. "
783
+ f"No sequence data found for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'. "
746
784
  "Is this a submission without sequence data?"
747
785
  )
748
786
  return self
@@ -767,7 +805,9 @@ class Donor(StrictBaseModel):
767
805
  and lab_datum.library_type in lib_types
768
806
  and not lab_datum.sequence_data.contains_files(FileType.bed)
769
807
  ):
770
- raise ValueError(f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'.")
808
+ raise ValueError(
809
+ f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'."
810
+ )
771
811
 
772
812
  return self
773
813
 
@@ -778,12 +818,14 @@ class Donor(StrictBaseModel):
778
818
  """
779
819
  for lab_datum in self.lab_data:
780
820
  if lab_datum.has_sequence_data() and not lab_datum.sequence_data.contains_files(FileType.vcf):
781
- raise ValueError(f"VCF file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.tan_g}'.")
821
+ raise ValueError(
822
+ f"VCF file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'."
823
+ )
782
824
 
783
825
  return self
784
826
 
785
827
  @model_validator(mode="after")
786
- def validate_fastq_file_exists(self):
828
+ def validate_fastq_file_exists(self): # noqa: C901
787
829
  """
788
830
  Check if there is a FASTQ file
789
831
  """
@@ -800,20 +842,44 @@ class Donor(StrictBaseModel):
800
842
  for i in fastq_files:
801
843
  if i.read_order is None:
802
844
  raise ValueError(
803
- f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.tan_g}': "
845
+ f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
804
846
  f"No read order specified for FASTQ file '{i.file_path}'!"
805
847
  )
806
848
 
807
- # check if there is an equal number of R1 and R2 files
808
- r1_fastq_files = [i for i in fastq_files if i.read_order == ReadOrder.r1]
809
- r2_fastq_files = [i for i in fastq_files if i.read_order == ReadOrder.r2]
849
+ key = lambda f: (f.flowcell_id, f.lane_id)
850
+ fastq_files.sort(key=key)
851
+ for _key, group in groupby(fastq_files, key):
852
+ flowcell_id = _key[0]
853
+ lane_id = _key[1]
854
+ files = list(group)
855
+
856
+ # separate R1 and R2 files
857
+ fastq_r1_files = [f for f in files if f.read_order == ReadOrder.r1]
858
+ fastq_r2_files = [f for f in files if f.read_order == ReadOrder.r2]
859
+
860
+ # check that there are exactly one R1 and on R2 file present
861
+ if len(fastq_r1_files) > 1:
862
+ raise ValueError(
863
+ f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
864
+ f"Paired end sequencing layout but multiple R1 files for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
865
+ )
866
+ elif len(fastq_r1_files) < 1:
867
+ raise ValueError(
868
+ f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
869
+ f"Paired end sequencing layout but missing R1 file for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
870
+ )
871
+
872
+ if len(fastq_r2_files) > 1:
873
+ raise ValueError(
874
+ f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
875
+ f"Paired end sequencing layout but multiple R2 files for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
876
+ )
877
+ elif len(fastq_r2_files) < 1:
878
+ raise ValueError(
879
+ f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
880
+ f"Paired end sequencing layout but missing R2 file for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
881
+ )
810
882
 
811
- if len(r1_fastq_files) != len(r2_fastq_files):
812
- raise ValueError(
813
- f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.tan_g}': "
814
- f"Paired end sequencing layout but number of R1 FASTQ files ({len(r1_fastq_files)})"
815
- f" differs from number of R2 FASTQ files ({len(r2_fastq_files)})!"
816
- )
817
883
  return self
818
884
 
819
885
 
@@ -822,6 +888,8 @@ class GrzSubmissionMetadata(StrictBaseModel):
822
888
  General metadata schema for submissions to the GRZ
823
889
  """
824
890
 
891
+ schema_: Annotated[str, Field(alias="$schema")] = SCHEMA_URL
892
+
825
893
  submission: Submission
826
894
 
827
895
  donors: list[Donor]
@@ -829,6 +897,12 @@ class GrzSubmissionMetadata(StrictBaseModel):
829
897
  List of donors including the index patient.
830
898
  """
831
899
 
900
+ @model_validator(mode="after")
901
+ def check_schema(self):
902
+ if self.schema_ != SCHEMA_URL:
903
+ log.warning(f"Unknown GRZ metadata schema URL: {self.schema_}")
904
+ return self
905
+
832
906
  @model_validator(mode="after")
833
907
  def validate_donor_count(self):
834
908
  """
@@ -858,11 +932,11 @@ class GrzSubmissionMetadata(StrictBaseModel):
858
932
  Check if oncology samples have tumor cell counts.
859
933
  """
860
934
  for donor in self.donors:
861
- case_id = donor.tan_g
935
+ pseudonym = donor.donor_pseudonym
862
936
  for lab_datum in donor.lab_data:
863
937
  if lab_datum.sequence_subtype == SequenceSubtype.somatic and lab_datum.tumor_cell_count is None:
864
938
  raise ValueError(
865
- f"Missing tumor cell count for donor '{case_id}', lab datum '{lab_datum.lab_data_name}'!"
939
+ f"Missing tumor cell count for donor '{pseudonym}', lab datum '{lab_datum.lab_data_name}'!"
866
940
  )
867
941
 
868
942
  return self
@@ -873,11 +947,11 @@ class GrzSubmissionMetadata(StrictBaseModel):
873
947
  Check if the submission contains lab data with the same name within one donor.
874
948
  """
875
949
  for donor in self.donors:
876
- case_id = donor.tan_g
950
+ pseudonym = donor.donor_pseudonym
877
951
  lab_data_names = set()
878
952
  for lab_datum in donor.lab_data:
879
953
  if lab_datum.lab_data_name in lab_data_names:
880
- raise ValueError(f"Duplicate lab datum '{lab_datum.lab_data_name}' in donor '{case_id}'")
954
+ raise ValueError(f"Duplicate lab datum '{lab_datum.lab_data_name}' in donor '{pseudonym}'")
881
955
  else:
882
956
  lab_data_names.add(lab_datum.lab_data_name)
883
957
 
@@ -908,7 +982,7 @@ class GrzSubmissionMetadata(StrictBaseModel):
908
982
  )
909
983
  info = dict(zip(names, key, strict=True))
910
984
  log.warning(
911
- f"No thresholds for the specified combination {info} found (donor {donor.tan_g})!\n"
985
+ f"No thresholds for the specified combination {info} found (donor {donor.donor_pseudonym})!\n"
912
986
  f"Valid combinations:\n{allowed_combinations}.\n"
913
987
  f"See https://www.bfarm.de/SharedDocs/Downloads/DE/Forschung/modellvorhaben-genomsequenzierung/Qs-durch-GRZ.pdf?__blob=publicationFile for more details.\n"
914
988
  f"Skipping threshold validation."
@@ -919,12 +993,29 @@ class GrzSubmissionMetadata(StrictBaseModel):
919
993
 
920
994
  return self
921
995
 
996
+ @model_validator(mode="after")
997
+ def validate_reference_genome_compatibility(self):
998
+ reference_genomes = {
999
+ (donor.donor_pseudonym, lab_datum.lab_data_name): lab_datum.sequence_data.reference_genome
1000
+ for donor in self.donors
1001
+ for lab_datum in donor.lab_data
1002
+ }
1003
+ unique_reference_genomes = set(reference_genomes.values())
1004
+ if len(unique_reference_genomes) > 1:
1005
+ raise ValueError(
1006
+ f"Incompatible reference genomes found: {unique_reference_genomes}.\n"
1007
+ f"Reference genomes must be consistent within a submission.\n"
1008
+ f"Reference genomes: {reference_genomes}"
1009
+ )
1010
+
1011
+ return self
1012
+
922
1013
 
923
1014
  def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, Any]):
924
1015
  if not lab_datum.has_sequence_data():
925
1016
  # Skip if no sequence data is present; warning issues in the validator `warn_empty_sequence_data` of `Donor`.
926
1017
  return
927
- case_id = donor.tan_g
1018
+ pseudonym = donor.donor_pseudonym
928
1019
  lab_data_name = lab_datum.lab_data_name
929
1020
  # mypy cannot reason about the `has_sequence_data` check
930
1021
  sequence_data = typing.cast(SequenceData, lab_datum.sequence_data)
@@ -933,17 +1024,18 @@ def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, A
933
1024
  mean_depth_of_coverage_v = sequence_data.mean_depth_of_coverage
934
1025
  if mean_depth_of_coverage_t and mean_depth_of_coverage_v < mean_depth_of_coverage_t:
935
1026
  raise ValueError(
936
- f"Mean depth of coverage for donor '{case_id}', lab datum '{lab_data_name}' "
1027
+ f"Mean depth of coverage for donor '{pseudonym}', lab datum '{lab_data_name}' "
937
1028
  f"below threshold: {mean_depth_of_coverage_v} < {mean_depth_of_coverage_t}"
938
1029
  )
939
1030
 
940
1031
  read_length_t = thresholds.get("readLength")
941
- read_length_v = sequence_data.read_length
942
- if read_length_t and read_length_v < read_length_t:
943
- raise ValueError(
944
- f"Read length for donor '{case_id}', lab datum '{lab_data_name}' "
945
- f"below threshold: {read_length_v} < {read_length_t}"
946
- )
1032
+ for f in sequence_data.list_files(FileType.fastq) + sequence_data.list_files(FileType.bam):
1033
+ read_length_v = f.read_length
1034
+ if read_length_t and read_length_v < read_length_t:
1035
+ raise ValueError(
1036
+ f"Read length for donor '{pseudonym}', lab datum '{lab_data_name}' "
1037
+ f"below threshold: {read_length_v} < {read_length_t}"
1038
+ )
947
1039
 
948
1040
  if t := thresholds.get("targetedRegionsAboveMinCoverage"):
949
1041
  min_coverage_t = t.get("minCoverage")
@@ -954,12 +1046,12 @@ def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, A
954
1046
 
955
1047
  if min_coverage_t and min_coverage_v < min_coverage_t:
956
1048
  raise ValueError(
957
- f"Minimum coverage for donor '{case_id}', lab datum '{lab_data_name}' "
1049
+ f"Minimum coverage for donor '{pseudonym}', lab datum '{lab_data_name}' "
958
1050
  f"below threshold: {min_coverage_v} < {min_coverage_t}"
959
1051
  )
960
1052
  if fraction_above_t and fraction_above_v < fraction_above_t:
961
1053
  raise ValueError(
962
- f"Fraction of targeted regions above minimum coverage for donor '{case_id}', "
1054
+ f"Fraction of targeted regions above minimum coverage for donor '{pseudonym}', "
963
1055
  f"lab datum '{lab_data_name}' below threshold: "
964
1056
  f"{fraction_above_v} < {fraction_above_t}"
965
1057
  )
@@ -15,7 +15,7 @@ from .download import S3BotoDownloadWorker
15
15
  from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
16
16
  from .file_operations import Crypt4GH, calculate_sha256
17
17
  from .models.config import ConfigModel
18
- from .models.v1_0_0.metadata import (
18
+ from .models.v1_1_1.metadata import (
19
19
  File,
20
20
  FileType,
21
21
  GrzSubmissionMetadata,
@@ -23,7 +23,7 @@ from .models.v1_0_0.metadata import (
23
23
  SequenceData,
24
24
  SequencingLayout,
25
25
  )
26
- from .models.v1_0_0.metadata import File as SubmissionFileMetadata
26
+ from .models.v1_1_1.metadata import File as SubmissionFileMetadata
27
27
  from .progress_logging import FileProgressLogger
28
28
  from .states import DecryptionState, EncryptionState, ValidationState
29
29
  from .upload import S3BotoUploadWorker
@@ -10,7 +10,7 @@ from os import PathLike
10
10
  from pathlib import Path
11
11
 
12
12
  from .file_operations import read_multiple_json
13
- from .models.v1_0_0.metadata import File as SubmissionFileMetadata
13
+ from .models.v1_1_1.metadata import File as SubmissionFileMetadata
14
14
  from .states import State
15
15
 
16
16
  Index = tuple[str, float, int]