variantgrid-api 1.2.0__tar.gz → 1.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {variantgrid_api-1.2.0/src/variantgrid_api.egg-info → variantgrid_api-1.3.1}/PKG-INFO +21 -1
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/README.md +20 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/pyproject.toml +4 -1
- variantgrid_api-1.3.1/src/variantgrid_api/api_client.py +419 -0
- variantgrid_api-1.3.1/src/variantgrid_api/cli.py +137 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api/data_models.py +31 -4
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api/mock_variantgrid_api.py +54 -3
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1/src/variantgrid_api.egg-info}/PKG-INFO +21 -1
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api.egg-info/SOURCES.txt +4 -0
- variantgrid_api-1.3.1/src/variantgrid_api.egg-info/entry_points.txt +2 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/tests/test_api_client.py +15 -4
- variantgrid_api-1.3.1/tests/test_api_client_annotation.py +221 -0
- variantgrid_api-1.3.1/tests/test_cli.py +98 -0
- variantgrid_api-1.3.1/tests/test_data_models.py +42 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/tests/test_mock_variantgrid_api.py +43 -0
- variantgrid_api-1.2.0/src/variantgrid_api/api_client.py +0 -251
- variantgrid_api-1.2.0/tests/test_data_models.py +0 -11
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/LICENSE +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/setup.cfg +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api.egg-info/dependency_links.txt +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api.egg-info/requires.txt +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/src/variantgrid_api.egg-info/top_level.txt +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/tests/test_api_client_bulk.py +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/tests/test_api_client_validation.py +0 -0
- {variantgrid_api-1.2.0 → variantgrid_api-1.3.1}/tests/test_sequencer_model_from_name.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: variantgrid_api
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.1
|
|
4
4
|
Summary: A Python API client for VariantGrid
|
|
5
5
|
Author-email: Dave Lawrence <davmlaw@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -66,6 +66,26 @@ enrichment_kit = EnrichmentKit(name="idt_haem", version=1)
|
|
|
66
66
|
result = api.create_enrichment_kit(enrichment_kit)
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
+
## Annotate a VCF and download it back
|
|
70
|
+
|
|
71
|
+
Upload a VCF, wait for VariantGrid to import + annotate any novel variants, then download the
|
|
72
|
+
cohort-level annotated export (all samples, single-sample VCFs included). The whole flow is a one-liner:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from variantgrid_api.api_client import VariantGridAPI
|
|
76
|
+
|
|
77
|
+
api = VariantGridAPI(server="https://variantgrid.com", api_token="YOUR_API_TOKEN")
|
|
78
|
+
|
|
79
|
+
# export_type is "vcf" (gzipped *.vcf.gz) or "csv" (zipped *.csv.zip)
|
|
80
|
+
path = api.annotate_vcf("input.vcf", export_type="vcf", dest_path="/data/results/")
|
|
81
|
+
print(f"Annotated VCF written to {path}")
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
There's also a `vg_api annotate_vcf` command line tool (first call uploads, run it again to download once
|
|
85
|
+
ready), a step-by-step form (`upload_file` → `wait_for_annotation` → `download_annotated`), and a "submit now,
|
|
86
|
+
download later" pattern for long-running jobs. See
|
|
87
|
+
**[Annotate a VCF](https://github.com/SACGF/variantgrid_api/wiki/Annotate-a-VCF)** on the wiki.
|
|
88
|
+
|
|
69
89
|
## Testing
|
|
70
90
|
|
|
71
91
|
```
|
|
@@ -23,6 +23,26 @@ enrichment_kit = EnrichmentKit(name="idt_haem", version=1)
|
|
|
23
23
|
result = api.create_enrichment_kit(enrichment_kit)
|
|
24
24
|
```
|
|
25
25
|
|
|
26
|
+
## Annotate a VCF and download it back
|
|
27
|
+
|
|
28
|
+
Upload a VCF, wait for VariantGrid to import + annotate any novel variants, then download the
|
|
29
|
+
cohort-level annotated export (all samples, single-sample VCFs included). The whole flow is a one-liner:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from variantgrid_api.api_client import VariantGridAPI
|
|
33
|
+
|
|
34
|
+
api = VariantGridAPI(server="https://variantgrid.com", api_token="YOUR_API_TOKEN")
|
|
35
|
+
|
|
36
|
+
# export_type is "vcf" (gzipped *.vcf.gz) or "csv" (zipped *.csv.zip)
|
|
37
|
+
path = api.annotate_vcf("input.vcf", export_type="vcf", dest_path="/data/results/")
|
|
38
|
+
print(f"Annotated VCF written to {path}")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
There's also a `vg_api annotate_vcf` command line tool (first call uploads, run it again to download once
|
|
42
|
+
ready), a step-by-step form (`upload_file` → `wait_for_annotation` → `download_annotated`), and a "submit now,
|
|
43
|
+
download later" pattern for long-running jobs. See
|
|
44
|
+
**[Annotate a VCF](https://github.com/SACGF/variantgrid_api/wiki/Annotate-a-VCF)** on the wiki.
|
|
45
|
+
|
|
26
46
|
## Testing
|
|
27
47
|
|
|
28
48
|
```
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "variantgrid_api"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.1"
|
|
8
8
|
description = "A Python API client for VariantGrid"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name = "Dave Lawrence", email = "davmlaw@gmail.com" }
|
|
@@ -23,6 +23,9 @@ dependencies = [
|
|
|
23
23
|
"dataclasses-json",
|
|
24
24
|
]
|
|
25
25
|
|
|
26
|
+
[project.scripts]
|
|
27
|
+
vg_api = "variantgrid_api.cli:main"
|
|
28
|
+
|
|
26
29
|
[project.optional-dependencies]
|
|
27
30
|
test = ["pytest>=8", "pytest-cov>=5", "responses"]
|
|
28
31
|
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
import urllib
|
|
7
|
+
import warnings
|
|
8
|
+
from enum import Enum
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import List, Optional, Callable, Union
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from variantgrid_api.data_models import EnrichmentKit, SequencingRun, SampleSheet, JointCalledVCF, \
|
|
15
|
+
SampleSheetLookup, SequencingFile, QCGeneList, QCExecStats, QCGeneCoverage, SequencerModel, Sequencer
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_UNSET = object()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DateTimeEncoder(json.JSONEncoder):
|
|
22
|
+
def default(self,o):
|
|
23
|
+
if isinstance(o,(datetime.date, datetime.datetime)):
|
|
24
|
+
return o.isoformat()
|
|
25
|
+
return super().default(o)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class EmptyInputPolicy(Enum):
|
|
29
|
+
IGNORE = "ignore"
|
|
30
|
+
WARN = "warn"
|
|
31
|
+
ERROR = "error"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AnnotationError(Exception):
|
|
35
|
+
"""Raised when the server reports an error while importing/annotating an uploaded file."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, message, status: Optional[dict] = None):
|
|
38
|
+
super().__init__(message)
|
|
39
|
+
self.status = status
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class VariantGridAPI:
|
|
43
|
+
def __init__(self, server, api_token,
|
|
44
|
+
empty_input_policy=EmptyInputPolicy.ERROR,
|
|
45
|
+
logger: Optional[logging.Logger] = None,
|
|
46
|
+
log_request=False, log_response=False
|
|
47
|
+
):
|
|
48
|
+
self.server = server
|
|
49
|
+
self.headers = {"Authorization": f"Token {api_token}"}
|
|
50
|
+
if logger is None:
|
|
51
|
+
logger = logging.getLogger(__name__)
|
|
52
|
+
self.logger = logger
|
|
53
|
+
self.validation_handler = self._get_validation_handler(empty_input_policy, logger)
|
|
54
|
+
self.log_request = log_request
|
|
55
|
+
self.log_response = log_response
|
|
56
|
+
|
|
57
|
+
def _get_url(self, url):
|
|
58
|
+
return urllib.parse.urljoin(self.server, url)
|
|
59
|
+
|
|
60
|
+
def _post(self, path, json_data):
|
|
61
|
+
url = self._get_url(path)
|
|
62
|
+
json_string = json.dumps(json_data, cls=DateTimeEncoder)
|
|
63
|
+
if self.log_request:
|
|
64
|
+
self.logger.info("POST to '%s', JSON: %s", url, json_string)
|
|
65
|
+
response = requests.post(url,
|
|
66
|
+
headers={**self.headers, "Content-Type": "application/json"},
|
|
67
|
+
data=json_string)
|
|
68
|
+
|
|
69
|
+
extra_error_message = f"{url=}, {json_string=}"
|
|
70
|
+
return self._handle_json_response(response, extra_error_message)
|
|
71
|
+
|
|
72
|
+
def _handle_json_response(self, response, extra_error_message: Optional[str] = None):
|
|
73
|
+
try:
|
|
74
|
+
json_response = response.json()
|
|
75
|
+
if self.log_response:
|
|
76
|
+
self.logger.info("Response from '%s', JSON: %s", response.url, json_response)
|
|
77
|
+
except Exception as e:
|
|
78
|
+
json_response = f"Couldn't convert JSON: {e}"
|
|
79
|
+
if not response.ok:
|
|
80
|
+
if extra_error_message:
|
|
81
|
+
self.logger.error(extra_error_message)
|
|
82
|
+
self.logger.error("Response: %s", json_response)
|
|
83
|
+
response.raise_for_status()
|
|
84
|
+
return json_response
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def _get_validation_handler(empty_input_policy: EmptyInputPolicy, logger: logging.Logger) -> Callable:
|
|
88
|
+
def _ignore(msg):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
def _warn(msg):
|
|
92
|
+
logger.warning(msg)
|
|
93
|
+
|
|
94
|
+
def _error(msg):
|
|
95
|
+
raise ValueError(msg)
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
EmptyInputPolicy.IGNORE: _ignore,
|
|
99
|
+
EmptyInputPolicy.WARN: _warn,
|
|
100
|
+
EmptyInputPolicy.ERROR: _error,
|
|
101
|
+
}[empty_input_policy]
|
|
102
|
+
|
|
103
|
+
def _validate_string(self, info: str, s: Optional[str]) -> None:
|
|
104
|
+
if s is None:
|
|
105
|
+
self.validation_handler(f"{info}: is None")
|
|
106
|
+
elif s == "":
|
|
107
|
+
self.validation_handler(f"{info}: empty string")
|
|
108
|
+
|
|
109
|
+
def _validate_object(self, info: str, obj):
|
|
110
|
+
if obj is None:
|
|
111
|
+
self.validation_handler(f"{info}: is None")
|
|
112
|
+
|
|
113
|
+
def _validate_list(self, info: str, list_obj: List):
|
|
114
|
+
if not list_obj:
|
|
115
|
+
self.validation_handler(f"{info}: empty list")
|
|
116
|
+
|
|
117
|
+
def create_experiment(self, experiment: str):
|
|
118
|
+
self._validate_string("experiment", experiment)
|
|
119
|
+
json_data = {
|
|
120
|
+
"name": experiment
|
|
121
|
+
}
|
|
122
|
+
return self._post("seqauto/api/v1/experiment/", json_data)
|
|
123
|
+
|
|
124
|
+
def create_enrichment_kit(self, enrichment_kit: EnrichmentKit):
|
|
125
|
+
self._validate_object("enrichment_kit", enrichment_kit)
|
|
126
|
+
return self._post("seqauto/api/v1/enrichment_kit/",
|
|
127
|
+
enrichment_kit.to_dict())
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def create_sequencer_model(self, sequencer_model: SequencerModel):
|
|
131
|
+
self._validate_object("sequencer_model", sequencer_model)
|
|
132
|
+
return self._post("seqauto/api/v1/sequencer_model/",
|
|
133
|
+
sequencer_model.to_dict())
|
|
134
|
+
|
|
135
|
+
def create_sequencer(self, sequencer: Sequencer):
|
|
136
|
+
self._validate_object("sequencer", sequencer)
|
|
137
|
+
return self._post("seqauto/api/v1/sequencer/",
|
|
138
|
+
sequencer.to_dict())
|
|
139
|
+
|
|
140
|
+
def create_sequencing_run(self, sequencing_run: SequencingRun):
|
|
141
|
+
self._validate_object("sequencing_run", sequencing_run)
|
|
142
|
+
return self._post("seqauto/api/v1/sequencing_run/",
|
|
143
|
+
sequencing_run.to_dict())
|
|
144
|
+
|
|
145
|
+
def create_sample_sheet(self, sample_sheet: SampleSheet):
|
|
146
|
+
self._validate_object("sample_sheet", sample_sheet)
|
|
147
|
+
json_data = sample_sheet.to_dict()
|
|
148
|
+
# We don't want all sequencing_run just the name
|
|
149
|
+
sequencing_run = json_data.pop("sequencing_run")
|
|
150
|
+
json_data["sequencing_run"] = sequencing_run["name"]
|
|
151
|
+
return self._post("seqauto/api/v1/sample_sheet/",
|
|
152
|
+
json_data)
|
|
153
|
+
|
|
154
|
+
def create_joint_called_vcf(self, joint_called_vcf: JointCalledVCF):
|
|
155
|
+
self._validate_object("joint_called_vcf", joint_called_vcf)
|
|
156
|
+
json_data = joint_called_vcf.to_dict()
|
|
157
|
+
return self._post("seqauto/api/v1/joint_called_vcf/",
|
|
158
|
+
json_data)
|
|
159
|
+
|
|
160
|
+
def create_sample_sheet_combined_vcf_file(self, sample_sheet_combined_vcf_file: JointCalledVCF):
|
|
161
|
+
"""Deprecated alias for :meth:`create_joint_called_vcf` - use that instead. """
|
|
162
|
+
warnings.warn(
|
|
163
|
+
"create_sample_sheet_combined_vcf_file is deprecated; use create_joint_called_vcf instead.",
|
|
164
|
+
DeprecationWarning,
|
|
165
|
+
stacklevel=2,
|
|
166
|
+
)
|
|
167
|
+
return self.create_joint_called_vcf(sample_sheet_combined_vcf_file)
|
|
168
|
+
|
|
169
|
+
def create_sequencing_data(self, sample_sheet_lookup: SampleSheetLookup, sequencing_files: List[SequencingFile]):
|
|
170
|
+
self._validate_object("sample_sheet_lookup", sample_sheet_lookup)
|
|
171
|
+
self._validate_list("sequencing_files", sequencing_files)
|
|
172
|
+
records = []
|
|
173
|
+
for sf in sequencing_files:
|
|
174
|
+
data = sf.to_dict()
|
|
175
|
+
# put into hierarchial JSON DRF expects
|
|
176
|
+
fastq_r1 = data.pop("fastq_r1")
|
|
177
|
+
fastq_r2 = data.pop("fastq_r2")
|
|
178
|
+
data["unaligned_reads"] = {
|
|
179
|
+
"fastq_r1": {"path": fastq_r1},
|
|
180
|
+
"fastq_r2": {"path": fastq_r2}
|
|
181
|
+
}
|
|
182
|
+
records.append(data)
|
|
183
|
+
|
|
184
|
+
json_data = {
|
|
185
|
+
"sample_sheet": sample_sheet_lookup.to_dict(),
|
|
186
|
+
"records": records
|
|
187
|
+
}
|
|
188
|
+
return self._post("seqauto/api/v1/sequencing_files/bulk_create",
|
|
189
|
+
json_data)
|
|
190
|
+
|
|
191
|
+
def create_qc_gene_list(self, qc_gene_list: QCGeneList):
|
|
192
|
+
self._validate_object("qc_gene_list", qc_gene_list)
|
|
193
|
+
json_data = qc_gene_list.to_dict()
|
|
194
|
+
return self._post("seqauto/api/v1/qc_gene_list/",
|
|
195
|
+
json_data)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def create_multiple_qc_gene_lists(self, qc_gene_lists: List[QCGeneList]):
|
|
199
|
+
self._validate_list("qc_gene_lists", qc_gene_lists)
|
|
200
|
+
json_data = {
|
|
201
|
+
"records": [
|
|
202
|
+
qcgl.to_dict() for qcgl in qc_gene_lists
|
|
203
|
+
]
|
|
204
|
+
}
|
|
205
|
+
return self._post("seqauto/api/v1/qc_gene_list/bulk_create",
|
|
206
|
+
json_data)
|
|
207
|
+
|
|
208
|
+
def create_qc_exec_stats(self, qc_exec_stats: QCExecStats):
|
|
209
|
+
self._validate_object("qc_exec_stats", qc_exec_stats)
|
|
210
|
+
json_data = qc_exec_stats.to_dict()
|
|
211
|
+
return self._post("seqauto/api/v1/qc_exec_summary/",
|
|
212
|
+
json_data)
|
|
213
|
+
|
|
214
|
+
def create_multiple_qc_exec_stats(self, qc_exec_stats: List[QCExecStats]):
|
|
215
|
+
self._validate_list("qc_exec_stats", qc_exec_stats)
|
|
216
|
+
json_data = {
|
|
217
|
+
"records": [
|
|
218
|
+
qces.to_dict() for qces in qc_exec_stats
|
|
219
|
+
]
|
|
220
|
+
}
|
|
221
|
+
return self._post("seqauto/api/v1/qc_exec_summary/bulk_create",
|
|
222
|
+
json_data)
|
|
223
|
+
|
|
224
|
+
def create_multiple_qc_gene_coverage(self, qc_gene_coverage_list: List[QCGeneCoverage]):
|
|
225
|
+
self._validate_list("qc_gene_coverage_list", qc_gene_coverage_list)
|
|
226
|
+
json_data = {
|
|
227
|
+
"records": [
|
|
228
|
+
qcgc.to_dict() for qcgc in qc_gene_coverage_list
|
|
229
|
+
]
|
|
230
|
+
}
|
|
231
|
+
return self._post("seqauto/api/v1/qc_gene_coverage/bulk_create",
|
|
232
|
+
json_data)
|
|
233
|
+
|
|
234
|
+
def upload_file(self, filename: str, path=_UNSET):
|
|
235
|
+
""" Upload a file via multipart POST to upload/api/v1/file_upload.
|
|
236
|
+
|
|
237
|
+
Returns {"uploaded_file_id": <id>, "sha256_hash": <hash>, ...}; identify the upload by
|
|
238
|
+
uploaded_file_id and/or sha256_hash (the server dedups on the content hash).
|
|
239
|
+
|
|
240
|
+
path: SeqAuto-only server-side hint that links the upload to a registered sequencing VCF
|
|
241
|
+
(JointCalledVCF / SingleSampleVCF) by path. Defaults to `filename` for backwards
|
|
242
|
+
compatibility. Pass path=None to omit the query param entirely - required for ad-hoc
|
|
243
|
+
uploads such as the annotate/download flow, where sending a client-side path makes
|
|
244
|
+
SeqAuto deployments try (and fail) to match it to a registered VCF. """
|
|
245
|
+
url = self._get_url("upload/api/v1/file_upload")
|
|
246
|
+
if path is _UNSET:
|
|
247
|
+
path = filename
|
|
248
|
+
params = {"path": path} if path is not None else {}
|
|
249
|
+
with open(filename, "rb") as f:
|
|
250
|
+
response = requests.post(url, headers=self.headers, files={"file": f}, params=params)
|
|
251
|
+
extra_error_message = f"{filename=}"
|
|
252
|
+
return self._handle_json_response(response, extra_error_message)
|
|
253
|
+
|
|
254
|
+
###############
|
|
255
|
+
## Get methods
|
|
256
|
+
|
|
257
|
+
def _get(self, path, params=None):
|
|
258
|
+
url = self._get_url(path)
|
|
259
|
+
response = requests.get(url,
|
|
260
|
+
params,
|
|
261
|
+
headers=self.headers)
|
|
262
|
+
|
|
263
|
+
extra_error_message = f"{url=}, {params=}"
|
|
264
|
+
return self._handle_json_response(response, extra_error_message)
|
|
265
|
+
|
|
266
|
+
def sequencing_run_has_vcf(self, sequencing_run: SequencingRun, path: Optional[str] = None):
|
|
267
|
+
""" Returns whether the SequencingRun has a VCF associated with it. If path is set, VCF must match that path
|
|
268
|
+
otherwise - any VCF present will retur True """
|
|
269
|
+
return self.sequencing_run_name_has_vcf(sequencing_run.name, path)
|
|
270
|
+
|
|
271
|
+
def sequencing_run_name_has_vcf(self, sequencing_run_name: str, path: Optional[str] = None):
|
|
272
|
+
""" Returns whether the SequencingRun has a VCF associated with it. If path is set, VCF must match that path
|
|
273
|
+
otherwise - any VCF present will retur True """
|
|
274
|
+
data = self._get(f"seqauto/api/v1/sequencing_run/{sequencing_run_name}/")
|
|
275
|
+
found_vcf = False
|
|
276
|
+
for vcf in data["vcf_set"]:
|
|
277
|
+
if path is None or path in vcf["path"]:
|
|
278
|
+
found_vcf = True
|
|
279
|
+
break
|
|
280
|
+
return found_vcf
|
|
281
|
+
|
|
282
|
+
##################################
|
|
283
|
+
## Uploaded file annotation flow
|
|
284
|
+
|
|
285
|
+
@staticmethod
|
|
286
|
+
def _upload_key_segment(uploaded_file_id: Optional[int], sha256: Optional[str]) -> str:
|
|
287
|
+
""" URL segment identifying an uploaded file - by uploaded_file_id (preferred) or its SHA-256 hash. """
|
|
288
|
+
if uploaded_file_id is not None:
|
|
289
|
+
return str(uploaded_file_id)
|
|
290
|
+
if sha256:
|
|
291
|
+
return f"sha256/{sha256}"
|
|
292
|
+
raise ValueError("Must provide one of 'uploaded_file_id' or 'sha256'")
|
|
293
|
+
|
|
294
|
+
def poll_upload_status(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None) -> dict:
|
|
295
|
+
""" Single GET of an uploaded file's import/annotation status.
|
|
296
|
+
|
|
297
|
+
Keyed by uploaded_file_id (returned from upload_file) or the SHA-256 of the uploaded file.
|
|
298
|
+
See wait_for_annotation to block until annotation is complete. """
|
|
299
|
+
segment = self._upload_key_segment(uploaded_file_id, sha256)
|
|
300
|
+
return self._get(f"upload/api/v1/upload_status/{segment}")
|
|
301
|
+
|
|
302
|
+
def wait_for_annotation(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None,
|
|
303
|
+
timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep,
|
|
304
|
+
max_transient_errors: int = 5) -> dict:
|
|
305
|
+
""" Poll poll_upload_status until 'annotation_complete' is true, then return the final status dict.
|
|
306
|
+
|
|
307
|
+
Raises AnnotationError if the server reports an 'error', or TimeoutError if 'timeout' seconds elapse.
|
|
308
|
+
'sleep' is injectable so tests can avoid real delays.
|
|
309
|
+
|
|
310
|
+
Transient server hiccups (5xx / connection / timeout - e.g. a brief 500 right after upload while the
|
|
311
|
+
server is still creating the upload record) are tolerated: up to 'max_transient_errors' *consecutive*
|
|
312
|
+
failures are retried before giving up. A 4xx response is treated as a real error and raised immediately.
|
|
313
|
+
The success counter resets whenever a poll succeeds. """
|
|
314
|
+
deadline = time.monotonic() + timeout
|
|
315
|
+
transient_errors = 0
|
|
316
|
+
while True:
|
|
317
|
+
try:
|
|
318
|
+
status = self.poll_upload_status(uploaded_file_id=uploaded_file_id, sha256=sha256)
|
|
319
|
+
transient_errors = 0
|
|
320
|
+
except (requests.HTTPError, requests.ConnectionError, requests.Timeout) as e:
|
|
321
|
+
status_code = getattr(getattr(e, "response", None), "status_code", None)
|
|
322
|
+
if status_code is not None and status_code < 500:
|
|
323
|
+
raise # 4xx is a real client error, not a transient blip
|
|
324
|
+
transient_errors += 1
|
|
325
|
+
if transient_errors > max_transient_errors:
|
|
326
|
+
raise
|
|
327
|
+
self.logger.warning("Transient error polling upload status (%s/%s), retrying: %s",
|
|
328
|
+
transient_errors, max_transient_errors, e)
|
|
329
|
+
if time.monotonic() >= deadline:
|
|
330
|
+
raise TimeoutError(f"Annotation did not complete within {timeout}s (last error: {e})")
|
|
331
|
+
sleep(poll_interval)
|
|
332
|
+
continue
|
|
333
|
+
if error := status.get("error"):
|
|
334
|
+
raise AnnotationError(error, status)
|
|
335
|
+
if status.get("annotation_complete"):
|
|
336
|
+
return status
|
|
337
|
+
if time.monotonic() >= deadline:
|
|
338
|
+
raise TimeoutError(f"Annotation did not complete within {timeout}s (status={status})")
|
|
339
|
+
sleep(poll_interval)
|
|
340
|
+
|
|
341
|
+
@staticmethod
|
|
342
|
+
def _attachment_filename(response, export_type: str) -> str:
|
|
343
|
+
content_disposition = response.headers.get("Content-Disposition", "")
|
|
344
|
+
if m := re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', content_disposition):
|
|
345
|
+
return urllib.parse.unquote(m.group(1))
|
|
346
|
+
ext = ".vcf.gz" if export_type == "vcf" else ".csv.zip"
|
|
347
|
+
return f"download{ext}"
|
|
348
|
+
|
|
349
|
+
def download_annotated(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None,
|
|
350
|
+
export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
|
|
351
|
+
timeout: float = 3600, poll_interval: float = 10,
|
|
352
|
+
sleep: Callable = time.sleep) -> Path:
|
|
353
|
+
""" Download the cohort-level annotated export of an uploaded VCF, saving it to disk.
|
|
354
|
+
|
|
355
|
+
export_type is 'vcf' (gzipped *.vcf.gz) or 'csv' (zipped *.csv.zip). The export covers all samples
|
|
356
|
+
in the uploaded VCF (single-sample VCFs included).
|
|
357
|
+
|
|
358
|
+
The endpoint returns 202 while the file is still being generated - this polls until it is ready (200),
|
|
359
|
+
then streams the attachment to dest_path. If dest_path is None the server's attachment filename is used
|
|
360
|
+
in the current directory; if it is an existing directory the attachment filename is placed inside it;
|
|
361
|
+
otherwise it is treated as the full destination path. Returns the Path written.
|
|
362
|
+
|
|
363
|
+
Raises TimeoutError if the file isn't ready within 'timeout' seconds. """
|
|
364
|
+
if export_type not in ("vcf", "csv"):
|
|
365
|
+
raise ValueError(f"export_type must be 'vcf' or 'csv', got {export_type!r}")
|
|
366
|
+
segment = self._upload_key_segment(uploaded_file_id, sha256)
|
|
367
|
+
url = self._get_url(f"upload/api/v1/download/{segment}/{export_type}")
|
|
368
|
+
deadline = time.monotonic() + timeout
|
|
369
|
+
while True:
|
|
370
|
+
response = requests.get(url, headers=self.headers, stream=True)
|
|
371
|
+
if response.status_code == 202:
|
|
372
|
+
if self.log_response:
|
|
373
|
+
self.logger.info("Download of '%s' still generating: %s", url, self._safe_json(response))
|
|
374
|
+
if time.monotonic() >= deadline:
|
|
375
|
+
raise TimeoutError(f"Download not ready within {timeout}s (url={url})")
|
|
376
|
+
sleep(poll_interval)
|
|
377
|
+
continue
|
|
378
|
+
if not response.ok:
|
|
379
|
+
# Raises with logging (JSON 'error' message from the server)
|
|
380
|
+
self._handle_json_response(response, f"download {export_type} for {segment}")
|
|
381
|
+
dest = self._resolve_download_dest(dest_path, response, export_type)
|
|
382
|
+
with open(dest, "wb") as f:
|
|
383
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
384
|
+
if chunk:
|
|
385
|
+
f.write(chunk)
|
|
386
|
+
return dest
|
|
387
|
+
|
|
388
|
+
def _resolve_download_dest(self, dest_path: Optional[Union[str, Path]], response, export_type: str) -> Path:
|
|
389
|
+
filename = self._attachment_filename(response, export_type)
|
|
390
|
+
if dest_path is None:
|
|
391
|
+
return Path(filename)
|
|
392
|
+
dest = Path(dest_path)
|
|
393
|
+
if dest.is_dir():
|
|
394
|
+
return dest / filename
|
|
395
|
+
return dest
|
|
396
|
+
|
|
397
|
+
@staticmethod
|
|
398
|
+
def _safe_json(response):
|
|
399
|
+
try:
|
|
400
|
+
return response.json()
|
|
401
|
+
except Exception:
|
|
402
|
+
return None
|
|
403
|
+
|
|
404
|
+
def annotate_vcf(self, filename: str, export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
|
|
405
|
+
timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep) -> Path:
|
|
406
|
+
""" Convenience one-liner: upload a VCF, wait for annotation to finish, download the annotated export.
|
|
407
|
+
|
|
408
|
+
Chains upload_file -> wait_for_annotation -> download_annotated and returns the Path written. """
|
|
409
|
+
# path is SeqAuto-only and makes ad-hoc uploads fail the import - omit it for the annotate flow
|
|
410
|
+
upload = self.upload_file(filename, path=None)
|
|
411
|
+
uploaded_file_id = upload["uploaded_file_id"]
|
|
412
|
+
self.wait_for_annotation(uploaded_file_id=uploaded_file_id,
|
|
413
|
+
timeout=timeout, poll_interval=poll_interval, sleep=sleep)
|
|
414
|
+
return self.download_annotated(uploaded_file_id=uploaded_file_id, export_type=export_type,
|
|
415
|
+
dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Command line interface for the VariantGrid API client.
|
|
2
|
+
|
|
3
|
+
Currently provides `vg_api annotate_vcf <file>`: a stateless upload/download helper. Because the server
|
|
4
|
+
dedups uploads on the file's SHA-256, the VCF file itself is the receipt - the first call uploads it, and
|
|
5
|
+
running the same command again downloads the annotated result (or reports that it isn't ready yet). No local
|
|
6
|
+
state (upload ids, pending.json, ...) is kept, so you can poll from any machine that has the VCF.
|
|
7
|
+
"""
|
|
8
|
+
import argparse
|
|
9
|
+
import hashlib
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from variantgrid_api.api_client import VariantGridAPI, AnnotationError
|
|
17
|
+
|
|
18
|
+
DEFAULT_SERVER = "https://variantgrid.com"
|
|
19
|
+
|
|
20
|
+
EXIT_OK = 0 # annotated file downloaded
|
|
21
|
+
EXIT_ERROR = 1 # bad input / auth / server or annotation error
|
|
22
|
+
EXIT_PENDING = 3 # uploaded just now, or still annotating - come back later
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _sha256(path):
|
|
26
|
+
"""Content hash the server dedups on - identical to `sha256sum <file>`."""
|
|
27
|
+
h = hashlib.sha256()
|
|
28
|
+
with open(path, "rb") as f:
|
|
29
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
30
|
+
h.update(chunk)
|
|
31
|
+
return h.hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _build_api(args):
|
|
35
|
+
token = args.token or os.environ.get("VARIANTGRID_API_TOKEN")
|
|
36
|
+
if not token:
|
|
37
|
+
raise SystemExit("No API token - pass --token or set VARIANTGRID_API_TOKEN")
|
|
38
|
+
server = args.server or os.environ.get("VARIANTGRID_API_SERVER") or DEFAULT_SERVER
|
|
39
|
+
# Own logger so we can silence the expected 404 when a file hasn't been uploaded yet.
|
|
40
|
+
logger = logging.getLogger("vg_api.cli")
|
|
41
|
+
return VariantGridAPI(server=server, api_token=token, logger=logger), logger
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _is_not_found(exc):
|
|
45
|
+
resp = getattr(exc, "response", None)
|
|
46
|
+
return resp is not None and resp.status_code == 404
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def annotate_vcf_cmd(args):
|
|
50
|
+
if not os.path.isfile(args.vcf):
|
|
51
|
+
print(f"No such file: {args.vcf}", file=sys.stderr)
|
|
52
|
+
return EXIT_ERROR
|
|
53
|
+
|
|
54
|
+
api, logger = _build_api(args)
|
|
55
|
+
name = os.path.basename(args.vcf)
|
|
56
|
+
|
|
57
|
+
if args.wait:
|
|
58
|
+
# Blocking one-shot: upload, wait (can take hours), download.
|
|
59
|
+
path = api.annotate_vcf(args.vcf, export_type=args.export_type, dest_path=args.dest,
|
|
60
|
+
poll_interval=args.poll_interval)
|
|
61
|
+
print(f"Annotated {args.export_type} written to {path}")
|
|
62
|
+
return EXIT_OK
|
|
63
|
+
|
|
64
|
+
sha256 = _sha256(args.vcf)
|
|
65
|
+
|
|
66
|
+
# Probe by content hash. A 404 means we've never uploaded this file - so upload it now.
|
|
67
|
+
prev_level = logger.level
|
|
68
|
+
logger.setLevel(logging.CRITICAL) # the probe 404 is expected; don't scare the user
|
|
69
|
+
try:
|
|
70
|
+
status = api.poll_upload_status(sha256=sha256)
|
|
71
|
+
except requests.HTTPError as e:
|
|
72
|
+
if _is_not_found(e):
|
|
73
|
+
up = api.upload_file(args.vcf, path=None)
|
|
74
|
+
print(f"Uploaded {name} (id={up['uploaded_file_id']}). "
|
|
75
|
+
f"Annotating - run the same command again later to download.")
|
|
76
|
+
return EXIT_PENDING
|
|
77
|
+
raise
|
|
78
|
+
finally:
|
|
79
|
+
logger.setLevel(prev_level)
|
|
80
|
+
|
|
81
|
+
if err := status.get("error"):
|
|
82
|
+
print(f"{name}: annotation error - {err}", file=sys.stderr)
|
|
83
|
+
return EXIT_ERROR
|
|
84
|
+
if status.get("annotation_complete"):
|
|
85
|
+
path = api.download_annotated(sha256=sha256, export_type=args.export_type, dest_path=args.dest)
|
|
86
|
+
print(f"Annotated {args.export_type} written to {path}")
|
|
87
|
+
return EXIT_OK
|
|
88
|
+
|
|
89
|
+
progress = status.get("progress_percent")
|
|
90
|
+
suffix = f" (progress {progress}%)" if progress is not None else ""
|
|
91
|
+
print(f"{name}: not ready yet{suffix} - run the same command again later.")
|
|
92
|
+
return EXIT_PENDING
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def build_parser():
|
|
96
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
97
|
+
common.add_argument("--server",
|
|
98
|
+
help="VariantGrid server URL (default: $VARIANTGRID_API_SERVER or "
|
|
99
|
+
f"{DEFAULT_SERVER})")
|
|
100
|
+
common.add_argument("--token", help="API token (default: $VARIANTGRID_API_TOKEN)")
|
|
101
|
+
|
|
102
|
+
parser = argparse.ArgumentParser(prog="vg_api", description="VariantGrid API command line tool")
|
|
103
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
104
|
+
|
|
105
|
+
p = sub.add_parser("annotate_vcf", parents=[common],
|
|
106
|
+
help="Upload a VCF for annotation; run again to download it once ready",
|
|
107
|
+
description="Upload a VCF for annotation. Because uploads are keyed on the file's "
|
|
108
|
+
"SHA-256, running the same command again downloads the annotated result "
|
|
109
|
+
"when ready, or reports that it isn't ready yet - no local state is kept.")
|
|
110
|
+
p.add_argument("vcf", help="Path to the VCF (.vcf / .vcf.gz) to annotate")
|
|
111
|
+
p.add_argument("--export-type", choices=("vcf", "csv"), default="vcf",
|
|
112
|
+
help="Download format (default: vcf)")
|
|
113
|
+
p.add_argument("-o", "--dest", default=".",
|
|
114
|
+
help="Destination directory or file for the download (default: current directory)")
|
|
115
|
+
p.add_argument("--wait", action="store_true",
|
|
116
|
+
help="Block until annotation finishes, then download (can take hours)")
|
|
117
|
+
p.add_argument("--poll-interval", type=float, default=10,
|
|
118
|
+
help="Seconds between status polls when using --wait (default: 10)")
|
|
119
|
+
p.set_defaults(func=annotate_vcf_cmd)
|
|
120
|
+
return parser
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def main(argv=None):
|
|
124
|
+
parser = build_parser()
|
|
125
|
+
args = parser.parse_args(argv)
|
|
126
|
+
try:
|
|
127
|
+
return args.func(args)
|
|
128
|
+
except AnnotationError as e:
|
|
129
|
+
print(f"Annotation error: {e}", file=sys.stderr)
|
|
130
|
+
return EXIT_ERROR
|
|
131
|
+
except requests.HTTPError as e:
|
|
132
|
+
print(f"HTTP error: {e}", file=sys.stderr)
|
|
133
|
+
return EXIT_ERROR
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
sys.exit(main())
|