grz-cli 0.1.4__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {grz_cli-0.1.4/src/grz_cli.egg-info → grz_cli-0.2.0}/PKG-INFO +1 -1
- {grz_cli-0.1.4 → grz_cli-0.2.0}/pyproject.toml +1 -1
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/cli.py +3 -2
- {grz_cli-0.1.4/src/grz_cli/models/v1_0_0 → grz_cli-0.2.0/src/grz_cli/models/v1_1_1}/metadata.py +226 -134
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/parser.py +2 -2
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/progress_logging.py +1 -1
- {grz_cli-0.1.4 → grz_cli-0.2.0/src/grz_cli.egg-info}/PKG-INFO +1 -1
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/SOURCES.txt +2 -4
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_file_progress_logging.py +1 -1
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_parser.py +5 -1
- grz_cli-0.1.4/LICENSE +0 -21
- grz_cli-0.1.4/src/grz_cli/resources/grz-schema.json +0 -595
- {grz_cli-0.1.4 → grz_cli-0.2.0}/MANIFEST.in +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/README.md +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/setup.cfg +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/__init__.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/__init__.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/constants.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/download.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/fastq_validation.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/file_operations.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/logging_setup.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/models/__init__.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/models/config.py +0 -0
- {grz_cli-0.1.4/src/grz_cli/models/v1_0_0 → grz_cli-0.2.0/src/grz_cli/models/v1_1_1}/__init__.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/resources/thresholds.json +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/states.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli/upload.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/dependency_links.txt +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/entry_points.txt +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/requires.txt +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/src/grz_cli.egg-info/top_level.txt +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_cli.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_download.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_fastq_validation.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_file_operations.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_read_multiple_json.py +0 -0
- {grz_cli-0.1.4 → grz_cli-0.2.0}/tests/test_upload.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: grz-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
|
|
5
5
|
Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
|
|
6
6
|
License: MIT
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
CLI module for handling command-line interface operations.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
+
import importlib
|
|
5
6
|
import logging
|
|
6
7
|
import logging.config
|
|
7
8
|
import sys
|
|
@@ -107,7 +108,7 @@ class OrderedGroup(click.Group):
|
|
|
107
108
|
cls=OrderedGroup,
|
|
108
109
|
help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
|
|
109
110
|
)
|
|
110
|
-
@click.version_option(version=
|
|
111
|
+
@click.version_option(version=importlib.metadata.version("grz-cli"), prog_name="grz-cli")
|
|
111
112
|
@click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
|
|
112
113
|
@click.option(
|
|
113
114
|
"--log-level",
|
|
@@ -229,7 +230,7 @@ def decrypt(
|
|
|
229
230
|
log.error("GRZ private key path is required for decryption.")
|
|
230
231
|
sys.exit(1)
|
|
231
232
|
|
|
232
|
-
log.info("Starting
|
|
233
|
+
log.info("Starting decryption...")
|
|
233
234
|
|
|
234
235
|
submission_dir = Path(submission_dir)
|
|
235
236
|
|
{grz_cli-0.1.4/src/grz_cli/models/v1_0_0 → grz_cli-0.2.0/src/grz_cli/models/v1_1_1}/metadata.py
RENAMED
|
@@ -1,19 +1,3 @@
|
|
|
1
|
-
# generated by datamodel-codegen:
|
|
2
|
-
# filename: grz-schema.json
|
|
3
|
-
# timestamp: 2024-11-21T12:40:39+00:00
|
|
4
|
-
# version: 0.26.2
|
|
5
|
-
### Command run:
|
|
6
|
-
## datamodel-codegen --output-model-type pydantic_v2.BaseModel \
|
|
7
|
-
## --input src/grz_cli/resources/grz-schema.json \
|
|
8
|
-
## --use-schema-description --use-field-description --snake-case-field --use-standard-collections --use-union-operator \
|
|
9
|
-
## --field-constraints --use-annotated --enable-version-header --reuse-model --use-subclass-enum --union-mode smart \
|
|
10
|
-
## --class-name 'GrzSubmissionMetadata' --use-double-quotes --target-python-version 3.12
|
|
11
|
-
### Custom modifications:
|
|
12
|
-
### - disallow extra fields (no additional properties allowed)
|
|
13
|
-
### - validate_assignment (i.e. when setting properties, re-trigger validations)
|
|
14
|
-
### - use_enum_values (use values [strings] instead of enum variants)
|
|
15
|
-
### - remove those aliases which are covered by a global `alias_generator=to_camel` setting
|
|
16
|
-
|
|
17
1
|
from __future__ import annotations
|
|
18
2
|
|
|
19
3
|
import json
|
|
@@ -23,6 +7,7 @@ from collections.abc import Generator
|
|
|
23
7
|
from datetime import date
|
|
24
8
|
from enum import StrEnum
|
|
25
9
|
from importlib.resources import files
|
|
10
|
+
from itertools import groupby
|
|
26
11
|
from pathlib import Path
|
|
27
12
|
from typing import Annotated, Any, Self
|
|
28
13
|
|
|
@@ -37,6 +22,8 @@ from pydantic.alias_generators import to_camel
|
|
|
37
22
|
|
|
38
23
|
from grz_cli.file_operations import calculate_sha256 # type: ignore
|
|
39
24
|
|
|
25
|
+
SCHEMA_URL = "https://raw.githubusercontent.com/BfArM-MVH/MVGenomseq/refs/tags/v1.1.1/GRZ/grz-schema.json"
|
|
26
|
+
|
|
40
27
|
log = logging.getLogger(__name__)
|
|
41
28
|
|
|
42
29
|
|
|
@@ -80,6 +67,33 @@ class GenomicStudySubtype(StrEnum):
|
|
|
80
67
|
germline_only = "germline-only"
|
|
81
68
|
|
|
82
69
|
|
|
70
|
+
class CoverageType(StrEnum):
|
|
71
|
+
"""
|
|
72
|
+
Health insurance providers
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
GKV = "GKV"
|
|
76
|
+
PKV = "PKV"
|
|
77
|
+
BG = "BG"
|
|
78
|
+
SEL = "SEL"
|
|
79
|
+
SOZ = "SOZ"
|
|
80
|
+
GPV = "GPV"
|
|
81
|
+
PPV = "PPV"
|
|
82
|
+
BEI = "BEI"
|
|
83
|
+
SKT = "SKT"
|
|
84
|
+
UNK = "UNK"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class DiseaseType(StrEnum):
|
|
88
|
+
"""
|
|
89
|
+
Type of the disease
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
oncological = "oncological"
|
|
93
|
+
rare = "rare"
|
|
94
|
+
hereditary = "hereditary"
|
|
95
|
+
|
|
96
|
+
|
|
83
97
|
class Submission(StrictBaseModel):
|
|
84
98
|
submission_date: date
|
|
85
99
|
"""
|
|
@@ -93,14 +107,19 @@ class Submission(StrictBaseModel):
|
|
|
93
107
|
|
|
94
108
|
tan_g: Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")]
|
|
95
109
|
"""
|
|
96
|
-
The VNg of the genomic data of the patient that will be reimbursed --> a unique 32-length byte code represented in a hex string of length 64.
|
|
110
|
+
The VNg of the genomic data of the index patient that will be reimbursed --> a unique 32-length byte code represented in a hex string of length 64.
|
|
97
111
|
"""
|
|
98
112
|
|
|
99
|
-
local_case_id: str
|
|
113
|
+
local_case_id: str
|
|
100
114
|
"""
|
|
101
115
|
A local case identifier for synchronizing locally
|
|
102
116
|
"""
|
|
103
117
|
|
|
118
|
+
coverage_type: CoverageType
|
|
119
|
+
"""
|
|
120
|
+
Health insurance provider
|
|
121
|
+
"""
|
|
122
|
+
|
|
104
123
|
submitter_id: Annotated[str, StringConstraints(pattern=r"^[0-9]{9}$")]
|
|
105
124
|
"""
|
|
106
125
|
Institutional ID of the submitter according to §293 SGB V.
|
|
@@ -119,6 +138,11 @@ class Submission(StrictBaseModel):
|
|
|
119
138
|
ID of the clinical data node in the format KDKXXXnnn.
|
|
120
139
|
"""
|
|
121
140
|
|
|
141
|
+
disease_type: DiseaseType
|
|
142
|
+
"""
|
|
143
|
+
Type of the disease
|
|
144
|
+
"""
|
|
145
|
+
|
|
122
146
|
genomic_study_type: GenomicStudyType
|
|
123
147
|
"""
|
|
124
148
|
whether additional persons are tested as well
|
|
@@ -156,49 +180,103 @@ class Relation(StrEnum):
|
|
|
156
180
|
brother = "brother"
|
|
157
181
|
sister = "sister"
|
|
158
182
|
child = "child"
|
|
159
|
-
|
|
183
|
+
index_ = "index"
|
|
160
184
|
other = "other"
|
|
161
185
|
|
|
162
186
|
|
|
187
|
+
class MvConsentScopeType(StrEnum):
|
|
188
|
+
"""
|
|
189
|
+
Consent or refusal to participate and consent, must be indicated for each option listed in the scope of consent.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
permit = "permit"
|
|
193
|
+
deny = "deny"
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class MvConsentScopeDomain(StrEnum):
|
|
197
|
+
"""
|
|
198
|
+
Scope of consent or revocation.
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
mv_sequencing = "mvSequencing"
|
|
202
|
+
re_identification = "reIdentification"
|
|
203
|
+
case_identification = "caseIdentification"
|
|
204
|
+
|
|
205
|
+
|
|
163
206
|
class MvConsentScope(StrictBaseModel):
|
|
164
207
|
"""
|
|
165
208
|
The scope of the Modellvorhaben consent given by the donor.
|
|
166
209
|
"""
|
|
167
210
|
|
|
168
|
-
|
|
211
|
+
type_: Annotated[MvConsentScopeType, Field(alias="type")]
|
|
212
|
+
"""
|
|
213
|
+
Consent or refusal to participate and consent, must be indicated for each option listed in the scope of consent.
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
date: date
|
|
217
|
+
"""
|
|
218
|
+
Date of signature of the pilot projects consent; in ISO 8601 format YYYY-MM-DD.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
domain: MvConsentScopeDomain
|
|
222
|
+
"""
|
|
223
|
+
Scope of consent or revocation.
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class MvConsent(StrictBaseModel):
|
|
228
|
+
presentation_date: date | None = None
|
|
169
229
|
"""
|
|
170
|
-
|
|
230
|
+
Date of delivery. Date (in ISO 8601 format YYYY-MM-DD) on which the Model Project Declaration of Participation
|
|
231
|
+
was presented to the patient, unless identical to the date of signature
|
|
171
232
|
"""
|
|
172
233
|
|
|
173
|
-
|
|
234
|
+
version: str
|
|
174
235
|
"""
|
|
175
|
-
|
|
236
|
+
Version of the declaration of participation.
|
|
237
|
+
Name and version of the declaration of participation in the MV GenomSeq, e.g.:
|
|
238
|
+
'Patient Info TE Consent MVGenomSeq vers01'
|
|
176
239
|
"""
|
|
177
240
|
|
|
178
|
-
|
|
241
|
+
scope: list[MvConsentScope]
|
|
179
242
|
"""
|
|
180
|
-
|
|
243
|
+
Modules of the consent to MV: must have at least a permit of mvSequencing
|
|
181
244
|
"""
|
|
182
245
|
|
|
246
|
+
@model_validator(mode="after")
|
|
247
|
+
def ensure_mv_sequencing_scope_is_present(self):
|
|
248
|
+
if not any(
|
|
249
|
+
scope.domain == MvConsentScopeDomain.mv_sequencing and scope.type_ == MvConsentScopeType.permit
|
|
250
|
+
for scope in self.scope
|
|
251
|
+
):
|
|
252
|
+
raise ValueError("Must have at least a permit of mvSequencing")
|
|
253
|
+
return self
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class ResearchConsentSchemaVersion(StrEnum):
|
|
257
|
+
v_2025_0_1 = "2025.0.1"
|
|
258
|
+
|
|
183
259
|
|
|
184
|
-
class
|
|
260
|
+
class ResearchConsent(StrictBaseModel):
|
|
185
261
|
"""
|
|
186
|
-
|
|
262
|
+
Research consents. Multiple declarations of consent are possible! Must be assigned to the respective data sets.
|
|
187
263
|
"""
|
|
188
264
|
|
|
189
|
-
|
|
265
|
+
schema_version: ResearchConsentSchemaVersion
|
|
190
266
|
"""
|
|
191
|
-
|
|
267
|
+
Schema version of de.medizininformatikinitiative.kerndatensatz.consent
|
|
192
268
|
"""
|
|
193
269
|
|
|
194
|
-
|
|
270
|
+
presentation_date: date | None = None
|
|
195
271
|
"""
|
|
196
|
-
|
|
272
|
+
Date of the delivery of the research consent in ISO 8601 format (YYYY-MM-DD)
|
|
197
273
|
"""
|
|
198
274
|
|
|
199
|
-
|
|
275
|
+
scope: object
|
|
200
276
|
"""
|
|
201
|
-
|
|
277
|
+
Scope of the research consent in JSON format following the MII IG Consent v2025 FHIR schema.
|
|
278
|
+
See 'https://www.medizininformatik-initiative.de/Kerndatensatz/KDS_Consent_V2025/MII-IG-Modul-Consent.html' and
|
|
279
|
+
'https://packages2.fhir.org/packages/de.medizininformatikinitiative.kerndatensatz.consent'.
|
|
202
280
|
"""
|
|
203
281
|
|
|
204
282
|
|
|
@@ -305,8 +383,8 @@ class TumorCellCountMethod(StrEnum):
|
|
|
305
383
|
Method used to determine cell count.
|
|
306
384
|
"""
|
|
307
385
|
|
|
308
|
-
pathology = "
|
|
309
|
-
bioinformatics = "
|
|
386
|
+
pathology = "pathology"
|
|
387
|
+
bioinformatics = "bioinformatics"
|
|
310
388
|
other = "other"
|
|
311
389
|
unknown = "unknown"
|
|
312
390
|
|
|
@@ -378,6 +456,11 @@ class File(StrictBaseModel):
|
|
|
378
456
|
Type of the file; if BED file is submitted, only 1 file is allowed.
|
|
379
457
|
"""
|
|
380
458
|
|
|
459
|
+
read_length: Annotated[int, Field(strict=True, ge=0)] | None = None
|
|
460
|
+
"""
|
|
461
|
+
The read length; in the case of long-read sequencing it is the rounded average read length.
|
|
462
|
+
"""
|
|
463
|
+
|
|
381
464
|
checksum_type: ChecksumType | None = ChecksumType.sha256
|
|
382
465
|
"""
|
|
383
466
|
Type of checksum algorithm used
|
|
@@ -408,6 +491,12 @@ class File(StrictBaseModel):
|
|
|
408
491
|
Indicates the lane
|
|
409
492
|
"""
|
|
410
493
|
|
|
494
|
+
@model_validator(mode="after")
|
|
495
|
+
def ensure_read_length_is_present_for_bam_and_fastq(self):
|
|
496
|
+
if self.file_type in {FileType.bam, FileType.fastq} and self.read_length is None:
|
|
497
|
+
raise ValueError(f"Read length missing for file '{self.file_path}' of type '{self.file_type}'.")
|
|
498
|
+
return self
|
|
499
|
+
|
|
411
500
|
def validate_data(self, local_file_path: Path) -> Generator[str]:
|
|
412
501
|
"""
|
|
413
502
|
Validates whether the provided file matches this metadata.
|
|
@@ -467,6 +556,15 @@ class PercentBasesAboveQualityThreshold(StrictBaseModel):
|
|
|
467
556
|
"""
|
|
468
557
|
|
|
469
558
|
|
|
559
|
+
class ReferenceGenome(StrEnum):
|
|
560
|
+
"""
|
|
561
|
+
Reference genome used according to the Genome Reference Consortium (https://www.ncbi.nlm.nih.gov/grc)
|
|
562
|
+
"""
|
|
563
|
+
|
|
564
|
+
GRCh37 = "GRCh37"
|
|
565
|
+
GRCh38 = "GRCh38"
|
|
566
|
+
|
|
567
|
+
|
|
470
568
|
class SequenceData(StrictBaseModel):
|
|
471
569
|
bioinformatics_pipeline_name: str
|
|
472
570
|
"""
|
|
@@ -478,9 +576,9 @@ class SequenceData(StrictBaseModel):
|
|
|
478
576
|
Version or commit hash of the bioinformatics pipeline
|
|
479
577
|
"""
|
|
480
578
|
|
|
481
|
-
reference_genome:
|
|
579
|
+
reference_genome: ReferenceGenome
|
|
482
580
|
"""
|
|
483
|
-
Reference genome used
|
|
581
|
+
Reference genome used according to the Genome Reference Consortium (https://www.ncbi.nlm.nih.gov/grc)
|
|
484
582
|
"""
|
|
485
583
|
|
|
486
584
|
percent_bases_above_quality_threshold: PercentBasesAboveQualityThreshold
|
|
@@ -503,11 +601,6 @@ class SequenceData(StrictBaseModel):
|
|
|
503
601
|
Fraction of targeted regions that are above minimum coverage
|
|
504
602
|
"""
|
|
505
603
|
|
|
506
|
-
read_length: Annotated[int, Field(strict=True, ge=0)]
|
|
507
|
-
"""
|
|
508
|
-
The read length; in the case of long-read sequencing it is the rounded average read length.
|
|
509
|
-
"""
|
|
510
|
-
|
|
511
604
|
non_coding_variants: bool
|
|
512
605
|
"""
|
|
513
606
|
The analysis includes non-coding variants -> true or false
|
|
@@ -652,14 +745,12 @@ class LabDatum(StrictBaseModel):
|
|
|
652
745
|
|
|
653
746
|
|
|
654
747
|
class Donor(StrictBaseModel):
|
|
655
|
-
|
|
656
|
-
"""
|
|
657
|
-
The VNg of the genomic data of the donor --> a unique 32-length byte code represented in a hex string of length 64.
|
|
658
|
-
"""
|
|
659
|
-
|
|
660
|
-
local_case_id: str | None = None
|
|
748
|
+
donor_pseudonym: str
|
|
661
749
|
"""
|
|
662
|
-
A
|
|
750
|
+
A unique identifier given by the Leistungserbringer for each donor of a single, duo or trio sequencing;
|
|
751
|
+
the donorPseudonym needs to be identifiable by the Leistungserbringer
|
|
752
|
+
in case of changes to the consents by one of the donors.
|
|
753
|
+
For Index patient use TanG.
|
|
663
754
|
"""
|
|
664
755
|
|
|
665
756
|
gender: Gender
|
|
@@ -672,64 +763,11 @@ class Donor(StrictBaseModel):
|
|
|
672
763
|
Relationship of the donor in respect to the index patient, e.g. 'index', 'brother', 'mother', etc.
|
|
673
764
|
"""
|
|
674
765
|
|
|
675
|
-
|
|
676
|
-
"""
|
|
677
|
-
Date of the signature of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
|
|
678
|
-
"""
|
|
679
|
-
|
|
680
|
-
mv_consent_presented_date: date | None = None
|
|
681
|
-
"""
|
|
682
|
-
Date of the presentation of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
|
|
683
|
-
"""
|
|
684
|
-
|
|
685
|
-
mv_consent_version: str
|
|
686
|
-
"""
|
|
687
|
-
Version of the research consent
|
|
688
|
-
"""
|
|
689
|
-
|
|
690
|
-
mv_consent_scope: MvConsentScope
|
|
691
|
-
"""
|
|
692
|
-
The scope of the Modellvorhaben consent given by the donor.
|
|
693
|
-
"""
|
|
694
|
-
|
|
695
|
-
mv_consent_termination_date: date | None = None
|
|
696
|
-
"""
|
|
697
|
-
Date of the termination of the Modellvorhaben consent; Date in ISO 8601 format YYYY-MM-DD
|
|
698
|
-
"""
|
|
699
|
-
|
|
700
|
-
mv_consent_termination_scope: MvConsentTerminationScope | None = None
|
|
701
|
-
"""
|
|
702
|
-
The scope of the termination of the Modellvorhaben consent.
|
|
703
|
-
"""
|
|
704
|
-
|
|
705
|
-
research_consent_date: date | None = None
|
|
706
|
-
"""
|
|
707
|
-
Date of the signature of the research consent; Date in ISO 8601 format YYYY-MM-DD
|
|
708
|
-
"""
|
|
709
|
-
|
|
710
|
-
research_consent_presented_date: date | None = None
|
|
711
|
-
"""
|
|
712
|
-
Date of the presentation of the research consent; Date in ISO 8601 format YYYY-MM-DD
|
|
713
|
-
"""
|
|
714
|
-
|
|
715
|
-
research_consent_version: str | None = None
|
|
716
|
-
"""
|
|
717
|
-
Version of the research consent
|
|
718
|
-
"""
|
|
719
|
-
|
|
720
|
-
research_consent_scope: dict[str, Any] | None = None
|
|
721
|
-
"""
|
|
722
|
-
Scope of the research consent given by the patient; shall be given in JSON format.
|
|
723
|
-
"""
|
|
724
|
-
|
|
725
|
-
research_consent_revocation_date: date | None = None
|
|
726
|
-
"""
|
|
727
|
-
Date of the revocation of the research consent; Date in ISO 8601 format YYYY-MM-DD
|
|
728
|
-
"""
|
|
766
|
+
mv_consent: MvConsent
|
|
729
767
|
|
|
730
|
-
|
|
768
|
+
research_consents: list[ResearchConsent]
|
|
731
769
|
"""
|
|
732
|
-
|
|
770
|
+
Research consents. Multiple declarations of consent are possible! Must be assigned to the respective data sets.
|
|
733
771
|
"""
|
|
734
772
|
|
|
735
773
|
lab_data: list[LabDatum]
|
|
@@ -742,7 +780,7 @@ class Donor(StrictBaseModel):
|
|
|
742
780
|
for lab_datum in self.lab_data:
|
|
743
781
|
if not lab_datum.has_sequence_data():
|
|
744
782
|
log.warning(
|
|
745
|
-
f"No sequence data found for lab datum '{lab_datum.lab_data_name}' in donor '{self.
|
|
783
|
+
f"No sequence data found for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'. "
|
|
746
784
|
"Is this a submission without sequence data?"
|
|
747
785
|
)
|
|
748
786
|
return self
|
|
@@ -767,7 +805,9 @@ class Donor(StrictBaseModel):
|
|
|
767
805
|
and lab_datum.library_type in lib_types
|
|
768
806
|
and not lab_datum.sequence_data.contains_files(FileType.bed)
|
|
769
807
|
):
|
|
770
|
-
raise ValueError(
|
|
808
|
+
raise ValueError(
|
|
809
|
+
f"BED file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'."
|
|
810
|
+
)
|
|
771
811
|
|
|
772
812
|
return self
|
|
773
813
|
|
|
@@ -778,12 +818,14 @@ class Donor(StrictBaseModel):
|
|
|
778
818
|
"""
|
|
779
819
|
for lab_datum in self.lab_data:
|
|
780
820
|
if lab_datum.has_sequence_data() and not lab_datum.sequence_data.contains_files(FileType.vcf):
|
|
781
|
-
raise ValueError(
|
|
821
|
+
raise ValueError(
|
|
822
|
+
f"VCF file missing for lab datum '{lab_datum.lab_data_name}' in donor '{self.donor_pseudonym}'."
|
|
823
|
+
)
|
|
782
824
|
|
|
783
825
|
return self
|
|
784
826
|
|
|
785
827
|
@model_validator(mode="after")
|
|
786
|
-
def validate_fastq_file_exists(self):
|
|
828
|
+
def validate_fastq_file_exists(self): # noqa: C901
|
|
787
829
|
"""
|
|
788
830
|
Check if there is a FASTQ file
|
|
789
831
|
"""
|
|
@@ -800,20 +842,44 @@ class Donor(StrictBaseModel):
|
|
|
800
842
|
for i in fastq_files:
|
|
801
843
|
if i.read_order is None:
|
|
802
844
|
raise ValueError(
|
|
803
|
-
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.
|
|
845
|
+
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
|
|
804
846
|
f"No read order specified for FASTQ file '{i.file_path}'!"
|
|
805
847
|
)
|
|
806
848
|
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
849
|
+
key = lambda f: (f.flowcell_id, f.lane_id)
|
|
850
|
+
fastq_files.sort(key=key)
|
|
851
|
+
for _key, group in groupby(fastq_files, key):
|
|
852
|
+
flowcell_id = _key[0]
|
|
853
|
+
lane_id = _key[1]
|
|
854
|
+
files = list(group)
|
|
855
|
+
|
|
856
|
+
# separate R1 and R2 files
|
|
857
|
+
fastq_r1_files = [f for f in files if f.read_order == ReadOrder.r1]
|
|
858
|
+
fastq_r2_files = [f for f in files if f.read_order == ReadOrder.r2]
|
|
859
|
+
|
|
860
|
+
# check that there are exactly one R1 and on R2 file present
|
|
861
|
+
if len(fastq_r1_files) > 1:
|
|
862
|
+
raise ValueError(
|
|
863
|
+
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
|
|
864
|
+
f"Paired end sequencing layout but multiple R1 files for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
|
|
865
|
+
)
|
|
866
|
+
elif len(fastq_r1_files) < 1:
|
|
867
|
+
raise ValueError(
|
|
868
|
+
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
|
|
869
|
+
f"Paired end sequencing layout but missing R1 file for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
|
|
870
|
+
)
|
|
871
|
+
|
|
872
|
+
if len(fastq_r2_files) > 1:
|
|
873
|
+
raise ValueError(
|
|
874
|
+
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
|
|
875
|
+
f"Paired end sequencing layout but multiple R2 files for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
|
|
876
|
+
)
|
|
877
|
+
elif len(fastq_r2_files) < 1:
|
|
878
|
+
raise ValueError(
|
|
879
|
+
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.donor_pseudonym}': "
|
|
880
|
+
f"Paired end sequencing layout but missing R2 file for flowcell id '{flowcell_id}', lane id '{lane_id}'!"
|
|
881
|
+
)
|
|
810
882
|
|
|
811
|
-
if len(r1_fastq_files) != len(r2_fastq_files):
|
|
812
|
-
raise ValueError(
|
|
813
|
-
f"Error in lab datum '{lab_datum.lab_data_name}' of donor '{self.tan_g}': "
|
|
814
|
-
f"Paired end sequencing layout but number of R1 FASTQ files ({len(r1_fastq_files)})"
|
|
815
|
-
f" differs from number of R2 FASTQ files ({len(r2_fastq_files)})!"
|
|
816
|
-
)
|
|
817
883
|
return self
|
|
818
884
|
|
|
819
885
|
|
|
@@ -822,6 +888,8 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
822
888
|
General metadata schema for submissions to the GRZ
|
|
823
889
|
"""
|
|
824
890
|
|
|
891
|
+
schema_: Annotated[str, Field(alias="$schema")] = SCHEMA_URL
|
|
892
|
+
|
|
825
893
|
submission: Submission
|
|
826
894
|
|
|
827
895
|
donors: list[Donor]
|
|
@@ -829,6 +897,12 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
829
897
|
List of donors including the index patient.
|
|
830
898
|
"""
|
|
831
899
|
|
|
900
|
+
@model_validator(mode="after")
|
|
901
|
+
def check_schema(self):
|
|
902
|
+
if self.schema_ != SCHEMA_URL:
|
|
903
|
+
log.warning(f"Unknown GRZ metadata schema URL: {self.schema_}")
|
|
904
|
+
return self
|
|
905
|
+
|
|
832
906
|
@model_validator(mode="after")
|
|
833
907
|
def validate_donor_count(self):
|
|
834
908
|
"""
|
|
@@ -858,11 +932,11 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
858
932
|
Check if oncology samples have tumor cell counts.
|
|
859
933
|
"""
|
|
860
934
|
for donor in self.donors:
|
|
861
|
-
|
|
935
|
+
pseudonym = donor.donor_pseudonym
|
|
862
936
|
for lab_datum in donor.lab_data:
|
|
863
937
|
if lab_datum.sequence_subtype == SequenceSubtype.somatic and lab_datum.tumor_cell_count is None:
|
|
864
938
|
raise ValueError(
|
|
865
|
-
f"Missing tumor cell count for donor '{
|
|
939
|
+
f"Missing tumor cell count for donor '{pseudonym}', lab datum '{lab_datum.lab_data_name}'!"
|
|
866
940
|
)
|
|
867
941
|
|
|
868
942
|
return self
|
|
@@ -873,11 +947,11 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
873
947
|
Check if the submission contains lab data with the same name within one donor.
|
|
874
948
|
"""
|
|
875
949
|
for donor in self.donors:
|
|
876
|
-
|
|
950
|
+
pseudonym = donor.donor_pseudonym
|
|
877
951
|
lab_data_names = set()
|
|
878
952
|
for lab_datum in donor.lab_data:
|
|
879
953
|
if lab_datum.lab_data_name in lab_data_names:
|
|
880
|
-
raise ValueError(f"Duplicate lab datum '{lab_datum.lab_data_name}' in donor '{
|
|
954
|
+
raise ValueError(f"Duplicate lab datum '{lab_datum.lab_data_name}' in donor '{pseudonym}'")
|
|
881
955
|
else:
|
|
882
956
|
lab_data_names.add(lab_datum.lab_data_name)
|
|
883
957
|
|
|
@@ -908,7 +982,7 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
908
982
|
)
|
|
909
983
|
info = dict(zip(names, key, strict=True))
|
|
910
984
|
log.warning(
|
|
911
|
-
f"No thresholds for the specified combination {info} found (donor {donor.
|
|
985
|
+
f"No thresholds for the specified combination {info} found (donor {donor.donor_pseudonym})!\n"
|
|
912
986
|
f"Valid combinations:\n{allowed_combinations}.\n"
|
|
913
987
|
f"See https://www.bfarm.de/SharedDocs/Downloads/DE/Forschung/modellvorhaben-genomsequenzierung/Qs-durch-GRZ.pdf?__blob=publicationFile for more details.\n"
|
|
914
988
|
f"Skipping threshold validation."
|
|
@@ -919,12 +993,29 @@ class GrzSubmissionMetadata(StrictBaseModel):
|
|
|
919
993
|
|
|
920
994
|
return self
|
|
921
995
|
|
|
996
|
+
@model_validator(mode="after")
|
|
997
|
+
def validate_reference_genome_compatibility(self):
|
|
998
|
+
reference_genomes = {
|
|
999
|
+
(donor.donor_pseudonym, lab_datum.lab_data_name): lab_datum.sequence_data.reference_genome
|
|
1000
|
+
for donor in self.donors
|
|
1001
|
+
for lab_datum in donor.lab_data
|
|
1002
|
+
}
|
|
1003
|
+
unique_reference_genomes = set(reference_genomes.values())
|
|
1004
|
+
if len(unique_reference_genomes) > 1:
|
|
1005
|
+
raise ValueError(
|
|
1006
|
+
f"Incompatible reference genomes found: {unique_reference_genomes}.\n"
|
|
1007
|
+
f"Reference genomes must be consistent within a submission.\n"
|
|
1008
|
+
f"Reference genomes: {reference_genomes}"
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
return self
|
|
1012
|
+
|
|
922
1013
|
|
|
923
1014
|
def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, Any]):
|
|
924
1015
|
if not lab_datum.has_sequence_data():
|
|
925
1016
|
# Skip if no sequence data is present; warning issues in the validator `warn_empty_sequence_data` of `Donor`.
|
|
926
1017
|
return
|
|
927
|
-
|
|
1018
|
+
pseudonym = donor.donor_pseudonym
|
|
928
1019
|
lab_data_name = lab_datum.lab_data_name
|
|
929
1020
|
# mypy cannot reason about the `has_sequence_data` check
|
|
930
1021
|
sequence_data = typing.cast(SequenceData, lab_datum.sequence_data)
|
|
@@ -933,17 +1024,18 @@ def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, A
|
|
|
933
1024
|
mean_depth_of_coverage_v = sequence_data.mean_depth_of_coverage
|
|
934
1025
|
if mean_depth_of_coverage_t and mean_depth_of_coverage_v < mean_depth_of_coverage_t:
|
|
935
1026
|
raise ValueError(
|
|
936
|
-
f"Mean depth of coverage for donor '{
|
|
1027
|
+
f"Mean depth of coverage for donor '{pseudonym}', lab datum '{lab_data_name}' "
|
|
937
1028
|
f"below threshold: {mean_depth_of_coverage_v} < {mean_depth_of_coverage_t}"
|
|
938
1029
|
)
|
|
939
1030
|
|
|
940
1031
|
read_length_t = thresholds.get("readLength")
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1032
|
+
for f in sequence_data.list_files(FileType.fastq) + sequence_data.list_files(FileType.bam):
|
|
1033
|
+
read_length_v = f.read_length
|
|
1034
|
+
if read_length_t and read_length_v < read_length_t:
|
|
1035
|
+
raise ValueError(
|
|
1036
|
+
f"Read length for donor '{pseudonym}', lab datum '{lab_data_name}' "
|
|
1037
|
+
f"below threshold: {read_length_v} < {read_length_t}"
|
|
1038
|
+
)
|
|
947
1039
|
|
|
948
1040
|
if t := thresholds.get("targetedRegionsAboveMinCoverage"):
|
|
949
1041
|
min_coverage_t = t.get("minCoverage")
|
|
@@ -954,12 +1046,12 @@ def _check_thresholds(donor: Donor, lab_datum: LabDatum, thresholds: dict[str, A
|
|
|
954
1046
|
|
|
955
1047
|
if min_coverage_t and min_coverage_v < min_coverage_t:
|
|
956
1048
|
raise ValueError(
|
|
957
|
-
f"Minimum coverage for donor '{
|
|
1049
|
+
f"Minimum coverage for donor '{pseudonym}', lab datum '{lab_data_name}' "
|
|
958
1050
|
f"below threshold: {min_coverage_v} < {min_coverage_t}"
|
|
959
1051
|
)
|
|
960
1052
|
if fraction_above_t and fraction_above_v < fraction_above_t:
|
|
961
1053
|
raise ValueError(
|
|
962
|
-
f"Fraction of targeted regions above minimum coverage for donor '{
|
|
1054
|
+
f"Fraction of targeted regions above minimum coverage for donor '{pseudonym}', "
|
|
963
1055
|
f"lab datum '{lab_data_name}' below threshold: "
|
|
964
1056
|
f"{fraction_above_v} < {fraction_above_t}"
|
|
965
1057
|
)
|
|
@@ -15,7 +15,7 @@ from .download import S3BotoDownloadWorker
|
|
|
15
15
|
from .fastq_validation import validate_paired_end_reads, validate_single_end_reads
|
|
16
16
|
from .file_operations import Crypt4GH, calculate_sha256
|
|
17
17
|
from .models.config import ConfigModel
|
|
18
|
-
from .models.
|
|
18
|
+
from .models.v1_1_1.metadata import (
|
|
19
19
|
File,
|
|
20
20
|
FileType,
|
|
21
21
|
GrzSubmissionMetadata,
|
|
@@ -23,7 +23,7 @@ from .models.v1_0_0.metadata import (
|
|
|
23
23
|
SequenceData,
|
|
24
24
|
SequencingLayout,
|
|
25
25
|
)
|
|
26
|
-
from .models.
|
|
26
|
+
from .models.v1_1_1.metadata import File as SubmissionFileMetadata
|
|
27
27
|
from .progress_logging import FileProgressLogger
|
|
28
28
|
from .states import DecryptionState, EncryptionState, ValidationState
|
|
29
29
|
from .upload import S3BotoUploadWorker
|
|
@@ -10,7 +10,7 @@ from os import PathLike
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
|
|
12
12
|
from .file_operations import read_multiple_json
|
|
13
|
-
from .models.
|
|
13
|
+
from .models.v1_1_1.metadata import File as SubmissionFileMetadata
|
|
14
14
|
from .states import State
|
|
15
15
|
|
|
16
16
|
Index = tuple[str, float, int]
|