variantgrid-api 1.1.1__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {variantgrid_api-1.1.1/src/variantgrid_api.egg-info → variantgrid_api-1.3.0}/PKG-INFO +39 -1
- variantgrid_api-1.3.0/README.md +70 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/pyproject.toml +1 -1
- variantgrid_api-1.3.0/src/variantgrid_api/api_client.py +419 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/src/variantgrid_api/data_models.py +55 -4
- variantgrid_api-1.3.0/src/variantgrid_api/mock_variantgrid_api.py +185 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0/src/variantgrid_api.egg-info}/PKG-INFO +39 -1
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/src/variantgrid_api.egg-info/SOURCES.txt +5 -1
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/tests/test_api_client.py +15 -4
- variantgrid_api-1.3.0/tests/test_api_client_annotation.py +221 -0
- variantgrid_api-1.3.0/tests/test_data_models.py +42 -0
- variantgrid_api-1.3.0/tests/test_mock_variantgrid_api.py +184 -0
- variantgrid_api-1.3.0/tests/test_sequencer_model_from_name.py +32 -0
- variantgrid_api-1.1.1/README.md +0 -32
- variantgrid_api-1.1.1/src/variantgrid_api/api_client.py +0 -251
- variantgrid_api-1.1.1/tests/test_data_models.py +0 -11
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/LICENSE +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/setup.cfg +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/src/variantgrid_api.egg-info/dependency_links.txt +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/src/variantgrid_api.egg-info/requires.txt +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/src/variantgrid_api.egg-info/top_level.txt +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/tests/test_api_client_bulk.py +0 -0
- {variantgrid_api-1.1.1 → variantgrid_api-1.3.0}/tests/test_api_client_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: variantgrid_api
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A Python API client for VariantGrid
|
|
5
5
|
Author-email: Dave Lawrence <davmlaw@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -66,6 +66,44 @@ enrichment_kit = EnrichmentKit(name="idt_haem", version=1)
|
|
|
66
66
|
result = api.create_enrichment_kit(enrichment_kit)
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
+
## Annotate a VCF and download it back
|
|
70
|
+
|
|
71
|
+
Upload a VCF, wait for VariantGrid to import + annotate any novel variants, then download the
|
|
72
|
+
cohort-level annotated export (all samples, single-sample VCFs included). The whole flow is a one-liner:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from variantgrid_api.api_client import VariantGridAPI
|
|
76
|
+
|
|
77
|
+
api = VariantGridAPI(server="https://variantgrid.com", api_token="YOUR_API_TOKEN")
|
|
78
|
+
|
|
79
|
+
# export_type is "vcf" (gzipped *.vcf.gz) or "csv" (zipped *.csv.zip)
|
|
80
|
+
path = api.annotate_vcf("input.vcf", export_type="vcf", dest_path="/data/results/")
|
|
81
|
+
print(f"Annotated VCF written to {path}")
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Or drive each step yourself:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
# For ad-hoc uploads pass path=None so the upload isn't treated as a SeqAuto backend-link hint
|
|
88
|
+
upload = api.upload_file("input.vcf", path=None)
|
|
89
|
+
uploaded_file_id = upload["uploaded_file_id"] # or upload["sha256_hash"]
|
|
90
|
+
api.wait_for_annotation(uploaded_file_id, timeout=3600, poll_interval=10) # raises on error/timeout
|
|
91
|
+
path = api.download_annotated(uploaded_file_id, export_type="csv", dest_path="/data/results/")
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
`poll_upload_status(uploaded_file_id)` returns the raw status dict (including `annotation_complete`,
|
|
95
|
+
`progress_percent`, `error`, `vcf_id`, `samples`, ...) if you want to inspect progress directly. All of
|
|
96
|
+
these accept `sha256=<hash>` instead of `uploaded_file_id` - the server dedups on the content hash, so it is
|
|
97
|
+
stable across machines and useful if you didn't retain the id.
|
|
98
|
+
|
|
99
|
+
Notes:
|
|
100
|
+
|
|
101
|
+
- The `annotate_vcf` wrapper already uploads with `path=None`. Only pass a `path` to `upload_file` for SeqAuto
|
|
102
|
+
uploads that link to a registered `JointCalledVCF` / `SingleSampleVCF` (that is what `path` is for - it is
|
|
103
|
+
ignored on non-SeqAuto deployments).
|
|
104
|
+
- Downloads require the target deployment to have the cohort export analysis templates configured
|
|
105
|
+
(`ANALYSIS_TEMPLATES_AUTO_COHORT_EXPORT`) - otherwise a clear error is returned.
|
|
106
|
+
|
|
69
107
|
## Testing
|
|
70
108
|
|
|
71
109
|
```
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# variantgrid_api
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/variantgrid_api/) [](https://pypi.org/project/variantgrid_api/)
|
|
4
|
+
|
|
5
|
+
Python API client for [VariantGrid](https://github.com/SACGF/variantgrid) Open source Variant database and analysis platform
|
|
6
|
+
|
|
7
|
+
See [changelog](https://github.com/SACGF/variantgrid_api/blob/main/CHANGELOG.md)
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
python3 -m pip install variantgrid_api
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Example
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
from variantgrid_api.api_client import VariantGridAPI
|
|
19
|
+
from variantgrid_api.data_models import EnrichmentKit
|
|
20
|
+
|
|
21
|
+
api = VariantGridAPI(server="https://variantgrid.com", api_token="YOUR_API_TOKEN")
|
|
22
|
+
enrichment_kit = EnrichmentKit(name="idt_haem", version=1)
|
|
23
|
+
result = api.create_enrichment_kit(enrichment_kit)
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Annotate a VCF and download it back
|
|
27
|
+
|
|
28
|
+
Upload a VCF, wait for VariantGrid to import + annotate any novel variants, then download the
|
|
29
|
+
cohort-level annotated export (all samples, single-sample VCFs included). The whole flow is a one-liner:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from variantgrid_api.api_client import VariantGridAPI
|
|
33
|
+
|
|
34
|
+
api = VariantGridAPI(server="https://variantgrid.com", api_token="YOUR_API_TOKEN")
|
|
35
|
+
|
|
36
|
+
# export_type is "vcf" (gzipped *.vcf.gz) or "csv" (zipped *.csv.zip)
|
|
37
|
+
path = api.annotate_vcf("input.vcf", export_type="vcf", dest_path="/data/results/")
|
|
38
|
+
print(f"Annotated VCF written to {path}")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Or drive each step yourself:
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
# For ad-hoc uploads pass path=None so the upload isn't treated as a SeqAuto backend-link hint
|
|
45
|
+
upload = api.upload_file("input.vcf", path=None)
|
|
46
|
+
uploaded_file_id = upload["uploaded_file_id"] # or upload["sha256_hash"]
|
|
47
|
+
api.wait_for_annotation(uploaded_file_id, timeout=3600, poll_interval=10) # raises on error/timeout
|
|
48
|
+
path = api.download_annotated(uploaded_file_id, export_type="csv", dest_path="/data/results/")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`poll_upload_status(uploaded_file_id)` returns the raw status dict (including `annotation_complete`,
|
|
52
|
+
`progress_percent`, `error`, `vcf_id`, `samples`, ...) if you want to inspect progress directly. All of
|
|
53
|
+
these accept `sha256=<hash>` instead of `uploaded_file_id` - the server dedups on the content hash, so it is
|
|
54
|
+
stable across machines and useful if you didn't retain the id.
|
|
55
|
+
|
|
56
|
+
Notes:
|
|
57
|
+
|
|
58
|
+
- The `annotate_vcf` wrapper already uploads with `path=None`. Only pass a `path` to `upload_file` for SeqAuto
|
|
59
|
+
uploads that link to a registered `JointCalledVCF` / `SingleSampleVCF` (that is what `path` is for - it is
|
|
60
|
+
ignored on non-SeqAuto deployments).
|
|
61
|
+
- Downloads require the target deployment to have the cohort export analysis templates configured
|
|
62
|
+
(`ANALYSIS_TEMPLATES_AUTO_COHORT_EXPORT`) - otherwise a clear error is returned.
|
|
63
|
+
|
|
64
|
+
## Testing
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
# Install required testing packages
|
|
68
|
+
python3 -m pip install -e ".[test]"
|
|
69
|
+
python3 -m pytest --cov=variantgrid_api
|
|
70
|
+
```
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
import urllib
|
|
7
|
+
import warnings
|
|
8
|
+
from enum import Enum
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import List, Optional, Callable, Union
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from variantgrid_api.data_models import EnrichmentKit, SequencingRun, SampleSheet, JointCalledVCF, \
|
|
15
|
+
SampleSheetLookup, SequencingFile, QCGeneList, QCExecStats, QCGeneCoverage, SequencerModel, Sequencer
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_UNSET = object()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DateTimeEncoder(json.JSONEncoder):
|
|
22
|
+
def default(self,o):
|
|
23
|
+
if isinstance(o,(datetime.date, datetime.datetime)):
|
|
24
|
+
return o.isoformat()
|
|
25
|
+
return super().default(o)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class EmptyInputPolicy(Enum):
|
|
29
|
+
IGNORE = "ignore"
|
|
30
|
+
WARN = "warn"
|
|
31
|
+
ERROR = "error"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AnnotationError(Exception):
|
|
35
|
+
"""Raised when the server reports an error while importing/annotating an uploaded file."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, message, status: Optional[dict] = None):
|
|
38
|
+
super().__init__(message)
|
|
39
|
+
self.status = status
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class VariantGridAPI:
|
|
43
|
+
def __init__(self, server, api_token,
|
|
44
|
+
empty_input_policy=EmptyInputPolicy.ERROR,
|
|
45
|
+
logger: Optional[logging.Logger] = None,
|
|
46
|
+
log_request=False, log_response=False
|
|
47
|
+
):
|
|
48
|
+
self.server = server
|
|
49
|
+
self.headers = {"Authorization": f"Token {api_token}"}
|
|
50
|
+
if logger is None:
|
|
51
|
+
logger = logging.getLogger(__name__)
|
|
52
|
+
self.logger = logger
|
|
53
|
+
self.validation_handler = self._get_validation_handler(empty_input_policy, logger)
|
|
54
|
+
self.log_request = log_request
|
|
55
|
+
self.log_response = log_response
|
|
56
|
+
|
|
57
|
+
def _get_url(self, url):
|
|
58
|
+
return urllib.parse.urljoin(self.server, url)
|
|
59
|
+
|
|
60
|
+
def _post(self, path, json_data):
|
|
61
|
+
url = self._get_url(path)
|
|
62
|
+
json_string = json.dumps(json_data, cls=DateTimeEncoder)
|
|
63
|
+
if self.log_request:
|
|
64
|
+
self.logger.info("POST to '%s', JSON: %s", url, json_string)
|
|
65
|
+
response = requests.post(url,
|
|
66
|
+
headers={**self.headers, "Content-Type": "application/json"},
|
|
67
|
+
data=json_string)
|
|
68
|
+
|
|
69
|
+
extra_error_message = f"{url=}, {json_string=}"
|
|
70
|
+
return self._handle_json_response(response, extra_error_message)
|
|
71
|
+
|
|
72
|
+
def _handle_json_response(self, response, extra_error_message: Optional[str] = None):
|
|
73
|
+
try:
|
|
74
|
+
json_response = response.json()
|
|
75
|
+
if self.log_response:
|
|
76
|
+
self.logger.info("Response from '%s', JSON: %s", response.url, json_response)
|
|
77
|
+
except Exception as e:
|
|
78
|
+
json_response = f"Couldn't convert JSON: {e}"
|
|
79
|
+
if not response.ok:
|
|
80
|
+
if extra_error_message:
|
|
81
|
+
self.logger.error(extra_error_message)
|
|
82
|
+
self.logger.error("Response: %s", json_response)
|
|
83
|
+
response.raise_for_status()
|
|
84
|
+
return json_response
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def _get_validation_handler(empty_input_policy: EmptyInputPolicy, logger: logging.Logger) -> Callable:
|
|
88
|
+
def _ignore(msg):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
def _warn(msg):
|
|
92
|
+
logger.warning(msg)
|
|
93
|
+
|
|
94
|
+
def _error(msg):
|
|
95
|
+
raise ValueError(msg)
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
EmptyInputPolicy.IGNORE: _ignore,
|
|
99
|
+
EmptyInputPolicy.WARN: _warn,
|
|
100
|
+
EmptyInputPolicy.ERROR: _error,
|
|
101
|
+
}[empty_input_policy]
|
|
102
|
+
|
|
103
|
+
def _validate_string(self, info: str, s: Optional[str]) -> None:
|
|
104
|
+
if s is None:
|
|
105
|
+
self.validation_handler(f"{info}: is None")
|
|
106
|
+
elif s == "":
|
|
107
|
+
self.validation_handler(f"{info}: empty string")
|
|
108
|
+
|
|
109
|
+
def _validate_object(self, info: str, obj):
|
|
110
|
+
if obj is None:
|
|
111
|
+
self.validation_handler(f"{info}: is None")
|
|
112
|
+
|
|
113
|
+
def _validate_list(self, info: str, list_obj: List):
|
|
114
|
+
if not list_obj:
|
|
115
|
+
self.validation_handler(f"{info}: empty list")
|
|
116
|
+
|
|
117
|
+
def create_experiment(self, experiment: str):
|
|
118
|
+
self._validate_string("experiment", experiment)
|
|
119
|
+
json_data = {
|
|
120
|
+
"name": experiment
|
|
121
|
+
}
|
|
122
|
+
return self._post("seqauto/api/v1/experiment/", json_data)
|
|
123
|
+
|
|
124
|
+
def create_enrichment_kit(self, enrichment_kit: EnrichmentKit):
|
|
125
|
+
self._validate_object("enrichment_kit", enrichment_kit)
|
|
126
|
+
return self._post("seqauto/api/v1/enrichment_kit/",
|
|
127
|
+
enrichment_kit.to_dict())
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def create_sequencer_model(self, sequencer_model: SequencerModel):
|
|
131
|
+
self._validate_object("sequencer_model", sequencer_model)
|
|
132
|
+
return self._post("seqauto/api/v1/sequencer_model/",
|
|
133
|
+
sequencer_model.to_dict())
|
|
134
|
+
|
|
135
|
+
def create_sequencer(self, sequencer: Sequencer):
|
|
136
|
+
self._validate_object("sequencer", sequencer)
|
|
137
|
+
return self._post("seqauto/api/v1/sequencer/",
|
|
138
|
+
sequencer.to_dict())
|
|
139
|
+
|
|
140
|
+
def create_sequencing_run(self, sequencing_run: SequencingRun):
|
|
141
|
+
self._validate_object("sequencing_run", sequencing_run)
|
|
142
|
+
return self._post("seqauto/api/v1/sequencing_run/",
|
|
143
|
+
sequencing_run.to_dict())
|
|
144
|
+
|
|
145
|
+
def create_sample_sheet(self, sample_sheet: SampleSheet):
|
|
146
|
+
self._validate_object("sample_sheet", sample_sheet)
|
|
147
|
+
json_data = sample_sheet.to_dict()
|
|
148
|
+
# We don't want all sequencing_run just the name
|
|
149
|
+
sequencing_run = json_data.pop("sequencing_run")
|
|
150
|
+
json_data["sequencing_run"] = sequencing_run["name"]
|
|
151
|
+
return self._post("seqauto/api/v1/sample_sheet/",
|
|
152
|
+
json_data)
|
|
153
|
+
|
|
154
|
+
def create_joint_called_vcf(self, joint_called_vcf: JointCalledVCF):
|
|
155
|
+
self._validate_object("joint_called_vcf", joint_called_vcf)
|
|
156
|
+
json_data = joint_called_vcf.to_dict()
|
|
157
|
+
return self._post("seqauto/api/v1/joint_called_vcf/",
|
|
158
|
+
json_data)
|
|
159
|
+
|
|
160
|
+
def create_sample_sheet_combined_vcf_file(self, sample_sheet_combined_vcf_file: JointCalledVCF):
|
|
161
|
+
"""Deprecated alias for :meth:`create_joint_called_vcf` - use that instead. """
|
|
162
|
+
warnings.warn(
|
|
163
|
+
"create_sample_sheet_combined_vcf_file is deprecated; use create_joint_called_vcf instead.",
|
|
164
|
+
DeprecationWarning,
|
|
165
|
+
stacklevel=2,
|
|
166
|
+
)
|
|
167
|
+
return self.create_joint_called_vcf(sample_sheet_combined_vcf_file)
|
|
168
|
+
|
|
169
|
+
def create_sequencing_data(self, sample_sheet_lookup: SampleSheetLookup, sequencing_files: List[SequencingFile]):
|
|
170
|
+
self._validate_object("sample_sheet_lookup", sample_sheet_lookup)
|
|
171
|
+
self._validate_list("sequencing_files", sequencing_files)
|
|
172
|
+
records = []
|
|
173
|
+
for sf in sequencing_files:
|
|
174
|
+
data = sf.to_dict()
|
|
175
|
+
# put into hierarchial JSON DRF expects
|
|
176
|
+
fastq_r1 = data.pop("fastq_r1")
|
|
177
|
+
fastq_r2 = data.pop("fastq_r2")
|
|
178
|
+
data["unaligned_reads"] = {
|
|
179
|
+
"fastq_r1": {"path": fastq_r1},
|
|
180
|
+
"fastq_r2": {"path": fastq_r2}
|
|
181
|
+
}
|
|
182
|
+
records.append(data)
|
|
183
|
+
|
|
184
|
+
json_data = {
|
|
185
|
+
"sample_sheet": sample_sheet_lookup.to_dict(),
|
|
186
|
+
"records": records
|
|
187
|
+
}
|
|
188
|
+
return self._post("seqauto/api/v1/sequencing_files/bulk_create",
|
|
189
|
+
json_data)
|
|
190
|
+
|
|
191
|
+
def create_qc_gene_list(self, qc_gene_list: QCGeneList):
|
|
192
|
+
self._validate_object("qc_gene_list", qc_gene_list)
|
|
193
|
+
json_data = qc_gene_list.to_dict()
|
|
194
|
+
return self._post("seqauto/api/v1/qc_gene_list/",
|
|
195
|
+
json_data)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def create_multiple_qc_gene_lists(self, qc_gene_lists: List[QCGeneList]):
|
|
199
|
+
self._validate_list("qc_gene_lists", qc_gene_lists)
|
|
200
|
+
json_data = {
|
|
201
|
+
"records": [
|
|
202
|
+
qcgl.to_dict() for qcgl in qc_gene_lists
|
|
203
|
+
]
|
|
204
|
+
}
|
|
205
|
+
return self._post("seqauto/api/v1/qc_gene_list/bulk_create",
|
|
206
|
+
json_data)
|
|
207
|
+
|
|
208
|
+
def create_qc_exec_stats(self, qc_exec_stats: QCExecStats):
|
|
209
|
+
self._validate_object("qc_exec_stats", qc_exec_stats)
|
|
210
|
+
json_data = qc_exec_stats.to_dict()
|
|
211
|
+
return self._post("seqauto/api/v1/qc_exec_summary/",
|
|
212
|
+
json_data)
|
|
213
|
+
|
|
214
|
+
def create_multiple_qc_exec_stats(self, qc_exec_stats: List[QCExecStats]):
|
|
215
|
+
self._validate_list("qc_exec_stats", qc_exec_stats)
|
|
216
|
+
json_data = {
|
|
217
|
+
"records": [
|
|
218
|
+
qces.to_dict() for qces in qc_exec_stats
|
|
219
|
+
]
|
|
220
|
+
}
|
|
221
|
+
return self._post("seqauto/api/v1/qc_exec_summary/bulk_create",
|
|
222
|
+
json_data)
|
|
223
|
+
|
|
224
|
+
def create_multiple_qc_gene_coverage(self, qc_gene_coverage_list: List[QCGeneCoverage]):
|
|
225
|
+
self._validate_list("qc_gene_coverage_list", qc_gene_coverage_list)
|
|
226
|
+
json_data = {
|
|
227
|
+
"records": [
|
|
228
|
+
qcgc.to_dict() for qcgc in qc_gene_coverage_list
|
|
229
|
+
]
|
|
230
|
+
}
|
|
231
|
+
return self._post("seqauto/api/v1/qc_gene_coverage/bulk_create",
|
|
232
|
+
json_data)
|
|
233
|
+
|
|
234
|
+
def upload_file(self, filename: str, path=_UNSET):
|
|
235
|
+
""" Upload a file via multipart POST to upload/api/v1/file_upload.
|
|
236
|
+
|
|
237
|
+
Returns {"uploaded_file_id": <id>, "sha256_hash": <hash>, ...}; identify the upload by
|
|
238
|
+
uploaded_file_id and/or sha256_hash (the server dedups on the content hash).
|
|
239
|
+
|
|
240
|
+
path: SeqAuto-only server-side hint that links the upload to a registered sequencing VCF
|
|
241
|
+
(JointCalledVCF / SingleSampleVCF) by path. Defaults to `filename` for backwards
|
|
242
|
+
compatibility. Pass path=None to omit the query param entirely - required for ad-hoc
|
|
243
|
+
uploads such as the annotate/download flow, where sending a client-side path makes
|
|
244
|
+
SeqAuto deployments try (and fail) to match it to a registered VCF. """
|
|
245
|
+
url = self._get_url("upload/api/v1/file_upload")
|
|
246
|
+
if path is _UNSET:
|
|
247
|
+
path = filename
|
|
248
|
+
params = {"path": path} if path is not None else {}
|
|
249
|
+
with open(filename, "rb") as f:
|
|
250
|
+
response = requests.post(url, headers=self.headers, files={"file": f}, params=params)
|
|
251
|
+
extra_error_message = f"{filename=}"
|
|
252
|
+
return self._handle_json_response(response, extra_error_message)
|
|
253
|
+
|
|
254
|
+
###############
|
|
255
|
+
## Get methods
|
|
256
|
+
|
|
257
|
+
def _get(self, path, params=None):
|
|
258
|
+
url = self._get_url(path)
|
|
259
|
+
response = requests.get(url,
|
|
260
|
+
params,
|
|
261
|
+
headers=self.headers)
|
|
262
|
+
|
|
263
|
+
extra_error_message = f"{url=}, {params=}"
|
|
264
|
+
return self._handle_json_response(response, extra_error_message)
|
|
265
|
+
|
|
266
|
+
def sequencing_run_has_vcf(self, sequencing_run: SequencingRun, path: Optional[str] = None):
|
|
267
|
+
""" Returns whether the SequencingRun has a VCF associated with it. If path is set, VCF must match that path
|
|
268
|
+
otherwise - any VCF present will retur True """
|
|
269
|
+
return self.sequencing_run_name_has_vcf(sequencing_run.name, path)
|
|
270
|
+
|
|
271
|
+
def sequencing_run_name_has_vcf(self, sequencing_run_name: str, path: Optional[str] = None):
|
|
272
|
+
""" Returns whether the SequencingRun has a VCF associated with it. If path is set, VCF must match that path
|
|
273
|
+
otherwise - any VCF present will retur True """
|
|
274
|
+
data = self._get(f"seqauto/api/v1/sequencing_run/{sequencing_run_name}/")
|
|
275
|
+
found_vcf = False
|
|
276
|
+
for vcf in data["vcf_set"]:
|
|
277
|
+
if path is None or path in vcf["path"]:
|
|
278
|
+
found_vcf = True
|
|
279
|
+
break
|
|
280
|
+
return found_vcf
|
|
281
|
+
|
|
282
|
+
##################################
|
|
283
|
+
## Uploaded file annotation flow
|
|
284
|
+
|
|
285
|
+
@staticmethod
|
|
286
|
+
def _upload_key_segment(uploaded_file_id: Optional[int], sha256: Optional[str]) -> str:
|
|
287
|
+
""" URL segment identifying an uploaded file - by uploaded_file_id (preferred) or its SHA-256 hash. """
|
|
288
|
+
if uploaded_file_id is not None:
|
|
289
|
+
return str(uploaded_file_id)
|
|
290
|
+
if sha256:
|
|
291
|
+
return f"sha256/{sha256}"
|
|
292
|
+
raise ValueError("Must provide one of 'uploaded_file_id' or 'sha256'")
|
|
293
|
+
|
|
294
|
+
def poll_upload_status(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None) -> dict:
|
|
295
|
+
""" Single GET of an uploaded file's import/annotation status.
|
|
296
|
+
|
|
297
|
+
Keyed by uploaded_file_id (returned from upload_file) or the SHA-256 of the uploaded file.
|
|
298
|
+
See wait_for_annotation to block until annotation is complete. """
|
|
299
|
+
segment = self._upload_key_segment(uploaded_file_id, sha256)
|
|
300
|
+
return self._get(f"upload/api/v1/upload_status/{segment}")
|
|
301
|
+
|
|
302
|
+
def wait_for_annotation(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None,
|
|
303
|
+
timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep,
|
|
304
|
+
max_transient_errors: int = 5) -> dict:
|
|
305
|
+
""" Poll poll_upload_status until 'annotation_complete' is true, then return the final status dict.
|
|
306
|
+
|
|
307
|
+
Raises AnnotationError if the server reports an 'error', or TimeoutError if 'timeout' seconds elapse.
|
|
308
|
+
'sleep' is injectable so tests can avoid real delays.
|
|
309
|
+
|
|
310
|
+
Transient server hiccups (5xx / connection / timeout - e.g. a brief 500 right after upload while the
|
|
311
|
+
server is still creating the upload record) are tolerated: up to 'max_transient_errors' *consecutive*
|
|
312
|
+
failures are retried before giving up. A 4xx response is treated as a real error and raised immediately.
|
|
313
|
+
The success counter resets whenever a poll succeeds. """
|
|
314
|
+
deadline = time.monotonic() + timeout
|
|
315
|
+
transient_errors = 0
|
|
316
|
+
while True:
|
|
317
|
+
try:
|
|
318
|
+
status = self.poll_upload_status(uploaded_file_id=uploaded_file_id, sha256=sha256)
|
|
319
|
+
transient_errors = 0
|
|
320
|
+
except (requests.HTTPError, requests.ConnectionError, requests.Timeout) as e:
|
|
321
|
+
status_code = getattr(getattr(e, "response", None), "status_code", None)
|
|
322
|
+
if status_code is not None and status_code < 500:
|
|
323
|
+
raise # 4xx is a real client error, not a transient blip
|
|
324
|
+
transient_errors += 1
|
|
325
|
+
if transient_errors > max_transient_errors:
|
|
326
|
+
raise
|
|
327
|
+
self.logger.warning("Transient error polling upload status (%s/%s), retrying: %s",
|
|
328
|
+
transient_errors, max_transient_errors, e)
|
|
329
|
+
if time.monotonic() >= deadline:
|
|
330
|
+
raise TimeoutError(f"Annotation did not complete within {timeout}s (last error: {e})")
|
|
331
|
+
sleep(poll_interval)
|
|
332
|
+
continue
|
|
333
|
+
if error := status.get("error"):
|
|
334
|
+
raise AnnotationError(error, status)
|
|
335
|
+
if status.get("annotation_complete"):
|
|
336
|
+
return status
|
|
337
|
+
if time.monotonic() >= deadline:
|
|
338
|
+
raise TimeoutError(f"Annotation did not complete within {timeout}s (status={status})")
|
|
339
|
+
sleep(poll_interval)
|
|
340
|
+
|
|
341
|
+
@staticmethod
|
|
342
|
+
def _attachment_filename(response, export_type: str) -> str:
|
|
343
|
+
content_disposition = response.headers.get("Content-Disposition", "")
|
|
344
|
+
if m := re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', content_disposition):
|
|
345
|
+
return urllib.parse.unquote(m.group(1))
|
|
346
|
+
ext = ".vcf.gz" if export_type == "vcf" else ".csv.zip"
|
|
347
|
+
return f"download{ext}"
|
|
348
|
+
|
|
349
|
+
def download_annotated(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None,
|
|
350
|
+
export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
|
|
351
|
+
timeout: float = 3600, poll_interval: float = 10,
|
|
352
|
+
sleep: Callable = time.sleep) -> Path:
|
|
353
|
+
""" Download the cohort-level annotated export of an uploaded VCF, saving it to disk.
|
|
354
|
+
|
|
355
|
+
export_type is 'vcf' (gzipped *.vcf.gz) or 'csv' (zipped *.csv.zip). The export covers all samples
|
|
356
|
+
in the uploaded VCF (single-sample VCFs included).
|
|
357
|
+
|
|
358
|
+
The endpoint returns 202 while the file is still being generated - this polls until it is ready (200),
|
|
359
|
+
then streams the attachment to dest_path. If dest_path is None the server's attachment filename is used
|
|
360
|
+
in the current directory; if it is an existing directory the attachment filename is placed inside it;
|
|
361
|
+
otherwise it is treated as the full destination path. Returns the Path written.
|
|
362
|
+
|
|
363
|
+
Raises TimeoutError if the file isn't ready within 'timeout' seconds. """
|
|
364
|
+
if export_type not in ("vcf", "csv"):
|
|
365
|
+
raise ValueError(f"export_type must be 'vcf' or 'csv', got {export_type!r}")
|
|
366
|
+
segment = self._upload_key_segment(uploaded_file_id, sha256)
|
|
367
|
+
url = self._get_url(f"upload/api/v1/download/{segment}/{export_type}")
|
|
368
|
+
deadline = time.monotonic() + timeout
|
|
369
|
+
while True:
|
|
370
|
+
response = requests.get(url, headers=self.headers, stream=True)
|
|
371
|
+
if response.status_code == 202:
|
|
372
|
+
if self.log_response:
|
|
373
|
+
self.logger.info("Download of '%s' still generating: %s", url, self._safe_json(response))
|
|
374
|
+
if time.monotonic() >= deadline:
|
|
375
|
+
raise TimeoutError(f"Download not ready within {timeout}s (url={url})")
|
|
376
|
+
sleep(poll_interval)
|
|
377
|
+
continue
|
|
378
|
+
if not response.ok:
|
|
379
|
+
# Raises with logging (JSON 'error' message from the server)
|
|
380
|
+
self._handle_json_response(response, f"download {export_type} for {segment}")
|
|
381
|
+
dest = self._resolve_download_dest(dest_path, response, export_type)
|
|
382
|
+
with open(dest, "wb") as f:
|
|
383
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
384
|
+
if chunk:
|
|
385
|
+
f.write(chunk)
|
|
386
|
+
return dest
|
|
387
|
+
|
|
388
|
+
def _resolve_download_dest(self, dest_path: Optional[Union[str, Path]], response, export_type: str) -> Path:
|
|
389
|
+
filename = self._attachment_filename(response, export_type)
|
|
390
|
+
if dest_path is None:
|
|
391
|
+
return Path(filename)
|
|
392
|
+
dest = Path(dest_path)
|
|
393
|
+
if dest.is_dir():
|
|
394
|
+
return dest / filename
|
|
395
|
+
return dest
|
|
396
|
+
|
|
397
|
+
@staticmethod
|
|
398
|
+
def _safe_json(response):
|
|
399
|
+
try:
|
|
400
|
+
return response.json()
|
|
401
|
+
except Exception:
|
|
402
|
+
return None
|
|
403
|
+
|
|
404
|
+
def annotate_vcf(self, filename: str, export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
|
|
405
|
+
timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep) -> Path:
|
|
406
|
+
""" Convenience one-liner: upload a VCF, wait for annotation to finish, download the annotated export.
|
|
407
|
+
|
|
408
|
+
Chains upload_file -> wait_for_annotation -> download_annotated and returns the Path written. """
|
|
409
|
+
# path is SeqAuto-only and makes ad-hoc uploads fail the import - omit it for the annotate flow
|
|
410
|
+
upload = self.upload_file(filename, path=None)
|
|
411
|
+
uploaded_file_id = upload["uploaded_file_id"]
|
|
412
|
+
self.wait_for_annotation(uploaded_file_id=uploaded_file_id,
|
|
413
|
+
timeout=timeout, poll_interval=poll_interval, sleep=sleep)
|
|
414
|
+
return self.download_annotated(uploaded_file_id=uploaded_file_id, export_type=export_type,
|
|
415
|
+
dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import re
|
|
2
|
+
import warnings
|
|
2
3
|
from datetime import date, datetime
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
from typing import Optional, List
|
|
@@ -25,6 +26,30 @@ class SequencerModel:
|
|
|
25
26
|
manufacturer: Manufacturer
|
|
26
27
|
data_naming_convention: str # 'M' or 'H' for (HiSeq only)
|
|
27
28
|
|
|
29
|
+
# (prefix, model_name, data_naming_convention)
|
|
30
|
+
_ILLUMINA_PREFIXES = [
|
|
31
|
+
("LH", "NovaSeq X", "M"),
|
|
32
|
+
("NB", "NextSeq", "M"),
|
|
33
|
+
("NS", "NextSeq", "M"),
|
|
34
|
+
("A", "NovaSeq 6000", "M"),
|
|
35
|
+
("H", "HiSeq", "H"),
|
|
36
|
+
("M", "MiSeq", "M"),
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_sequencer_name(cls, sequencer_name: str) -> 'SequencerModel':
|
|
41
|
+
"""Infer SequencerModel from an Illumina sequencer instrument name.
|
|
42
|
+
|
|
43
|
+
Illumina instrument names begin with a letter code that identifies the
|
|
44
|
+
platform (e.g. ``M02027`` → MiSeq, ``A01234`` → NovaSeq 6000).
|
|
45
|
+
Returns an Unknown model when no prefix matches.
|
|
46
|
+
"""
|
|
47
|
+
illumina = Manufacturer(name="Illumina")
|
|
48
|
+
for prefix, model_name, convention in cls._ILLUMINA_PREFIXES:
|
|
49
|
+
if sequencer_name.startswith(prefix):
|
|
50
|
+
return cls(model=model_name, manufacturer=illumina, data_naming_convention=convention)
|
|
51
|
+
return cls(model="Unknown", manufacturer=illumina, data_naming_convention="U")
|
|
52
|
+
|
|
28
53
|
|
|
29
54
|
@dataclass_json
|
|
30
55
|
@dataclass
|
|
@@ -109,12 +134,25 @@ class VariantCaller:
|
|
|
109
134
|
|
|
110
135
|
@dataclass_json
|
|
111
136
|
@dataclass
|
|
112
|
-
class
|
|
137
|
+
class JointCalledVCF:
|
|
138
|
+
"""A joint-called multi-sample VCF (mirrors the server ``JointCalledVCF`` model). """
|
|
113
139
|
path: str
|
|
114
140
|
sample_sheet_lookup: SampleSheetLookup = field(metadata=config(field_name="sample_sheet"))
|
|
115
141
|
variant_caller: VariantCaller
|
|
116
142
|
|
|
117
143
|
|
|
144
|
+
@dataclass_json
|
|
145
|
+
@dataclass
|
|
146
|
+
class SampleSheetCombinedVCFFile(JointCalledVCF):
|
|
147
|
+
"""Deprecated alias for :class:`JointCalledVCF` - use that instead. """
|
|
148
|
+
def __post_init__(self):
|
|
149
|
+
warnings.warn(
|
|
150
|
+
"SampleSheetCombinedVCFFile is deprecated; use JointCalledVCF instead.",
|
|
151
|
+
DeprecationWarning,
|
|
152
|
+
stacklevel=2,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
118
156
|
@dataclass_json
|
|
119
157
|
@dataclass
|
|
120
158
|
class BamFile:
|
|
@@ -124,11 +162,24 @@ class BamFile:
|
|
|
124
162
|
|
|
125
163
|
@dataclass_json
|
|
126
164
|
@dataclass
|
|
127
|
-
class
|
|
165
|
+
class SingleSampleVCF:
|
|
166
|
+
"""A per-sample VCF - one BAM in, one VCF out (mirrors the server ``SingleSampleVCF`` model). """
|
|
128
167
|
path: str
|
|
129
168
|
variant_caller: Optional[VariantCaller] = field(default=None, metadata=config(exclude=lambda x: x is None))
|
|
130
169
|
|
|
131
170
|
|
|
171
|
+
@dataclass_json
|
|
172
|
+
@dataclass
|
|
173
|
+
class VCFFile(SingleSampleVCF):
|
|
174
|
+
"""Deprecated alias for :class:`SingleSampleVCF` - use that instead. """
|
|
175
|
+
def __post_init__(self):
|
|
176
|
+
warnings.warn(
|
|
177
|
+
"VCFFile is deprecated; use SingleSampleVCF instead.",
|
|
178
|
+
DeprecationWarning,
|
|
179
|
+
stacklevel=2,
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
|
|
132
183
|
@dataclass_json
|
|
133
184
|
@dataclass
|
|
134
185
|
class SequencingFile:
|
|
@@ -136,7 +187,7 @@ class SequencingFile:
|
|
|
136
187
|
fastq_r1: str
|
|
137
188
|
fastq_r2: str
|
|
138
189
|
bam_file: BamFile
|
|
139
|
-
vcf_file:
|
|
190
|
+
vcf_file: SingleSampleVCF
|
|
140
191
|
|
|
141
192
|
|
|
142
193
|
@dataclass_json
|
|
@@ -155,7 +206,7 @@ class QC:
|
|
|
155
206
|
"""
|
|
156
207
|
sequencing_sample_lookup: SequencingSampleLookup = field(metadata=config(field_name="sequencing_sample"))
|
|
157
208
|
bam_file: BamFile
|
|
158
|
-
vcf_file:
|
|
209
|
+
vcf_file: SingleSampleVCF
|
|
159
210
|
|
|
160
211
|
|
|
161
212
|
@dataclass_json
|