pori-python 1.5.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pori_python-1.5.0/pori_python.egg-info → pori_python-1.6.0}/PKG-INFO +4 -1
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/match.py +2 -27
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/util.py +125 -39
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/vocab.py +9 -15
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/content.spec.json +4 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/ipr.py +29 -9
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/main.py +33 -23
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/summary.py +6 -6
- {pori_python-1.5.0 → pori_python-1.6.0/pori_python.egg-info}/PKG-INFO +4 -1
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python.egg-info/SOURCES.txt +1 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python.egg-info/requires.txt +3 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/setup.cfg +8 -1
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/test_match.py +0 -16
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/test_statement.py +14 -3
- pori_python-1.6.0/tests/test_graphkb/test_util.py +393 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_annotate.py +25 -1
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_ipr.py +6 -0
- pori_python-1.6.0/tests/test_ipr/test_ipr_integration.py +102 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_main.py +0 -1
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_summary.py +32 -32
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_upload.py +3 -3
- pori_python-1.5.0/tests/test_graphkb/test_util.py +0 -193
- {pori_python-1.5.0 → pori_python-1.6.0}/LICENSE +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/MANIFEST.in +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/README.md +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/constants.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/genes.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/graphkb/statement.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/annotate.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/connection.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/constants.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/inputs.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/therapeutic_options.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/ipr/util.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/py.typed +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python/types.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python.egg-info/dependency_links.txt +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python.egg-info/entry_points.txt +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pori_python.egg-info/top_level.txt +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/pyproject.toml +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/setup.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/data.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/test_genes.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/test_graphkb.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_graphkb/test_vocab.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/__init__.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/constants.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_connection.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_inputs.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_probe.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/test_util.py +0 -0
- {pori_python-1.5.0 → pori_python-1.6.0}/tests/test_ipr/util.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pori_python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Home-page: https://github.com/bcgsc/pori_python
|
|
5
5
|
Author-email: dat@bcgsc.ca
|
|
6
6
|
Maintainer-email: dat@bcgsc.ca
|
|
@@ -12,6 +12,9 @@ Requires-Dist: pandas>=1.1.0
|
|
|
12
12
|
Requires-Dist: requests
|
|
13
13
|
Requires-Dist: tqdm
|
|
14
14
|
Requires-Dist: typing_extensions<5,>=3.7.4.2
|
|
15
|
+
Requires-Dist: requests-cache<2.0,>=1.3.0
|
|
16
|
+
Requires-Dist: requests-ratelimiter
|
|
17
|
+
Requires-Dist: pyrate-limiter
|
|
15
18
|
Provides-Extra: deploy
|
|
16
19
|
Requires-Dist: twine; extra == "deploy"
|
|
17
20
|
Requires-Dist: wheel; extra == "deploy"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Functions which return Variants from GraphKB which match some input variant definition
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from typing import Dict, List, Optional,
|
|
5
|
+
from typing import Dict, List, Optional, Union, cast
|
|
6
6
|
|
|
7
7
|
from pori_python.types import (
|
|
8
8
|
BasicPosition,
|
|
@@ -31,14 +31,7 @@ from .util import (
|
|
|
31
31
|
looks_like_rid,
|
|
32
32
|
stringifyVariant,
|
|
33
33
|
)
|
|
34
|
-
from .vocab import
|
|
35
|
-
get_equivalent_terms,
|
|
36
|
-
get_term_by_name,
|
|
37
|
-
get_term_tree,
|
|
38
|
-
get_terms_set,
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
FEATURES_CACHE: Set[str] = set()
|
|
34
|
+
from .vocab import get_equivalent_terms, get_term_by_name, get_term_tree, get_terms_set
|
|
42
35
|
|
|
43
36
|
|
|
44
37
|
def get_equivalent_features(
|
|
@@ -97,8 +90,6 @@ def get_equivalent_features(
|
|
|
97
90
|
filters.append(
|
|
98
91
|
{'OR': [{'sourceIdVersion': source_id_version}, {'sourceIdVersion': None}]}
|
|
99
92
|
)
|
|
100
|
-
elif FEATURES_CACHE and gene_name.lower() not in FEATURES_CACHE and not ignore_cache:
|
|
101
|
-
return []
|
|
102
93
|
else:
|
|
103
94
|
filters.append({'OR': [{'sourceId': gene_name}, {'name': gene_name}]})
|
|
104
95
|
|
|
@@ -111,22 +102,6 @@ def get_equivalent_features(
|
|
|
111
102
|
)
|
|
112
103
|
|
|
113
104
|
|
|
114
|
-
def cache_missing_features(conn: GraphKBConnection) -> None:
|
|
115
|
-
"""
|
|
116
|
-
Create a cache of features that exist to avoid repeatedly querying
|
|
117
|
-
for missing features
|
|
118
|
-
"""
|
|
119
|
-
genes = cast(
|
|
120
|
-
List[Ontology],
|
|
121
|
-
conn.query({'target': 'Feature', 'returnProperties': ['name', 'sourceId'], 'neighbors': 0}),
|
|
122
|
-
)
|
|
123
|
-
for gene in genes:
|
|
124
|
-
if gene['name']:
|
|
125
|
-
FEATURES_CACHE.add(gene['name'].lower())
|
|
126
|
-
if gene['sourceId']:
|
|
127
|
-
FEATURES_CACHE.add(gene['sourceId'].lower())
|
|
128
|
-
|
|
129
|
-
|
|
130
105
|
def match_category_variant(
|
|
131
106
|
conn: GraphKBConnection,
|
|
132
107
|
reference_name: str,
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
from requests.adapters import HTTPAdapter
|
|
3
|
-
|
|
4
|
-
import hashlib
|
|
5
1
|
import json
|
|
6
2
|
import logging
|
|
7
3
|
import os
|
|
8
4
|
import re
|
|
9
5
|
import time
|
|
10
6
|
from datetime import datetime
|
|
11
|
-
from typing import
|
|
12
|
-
from urllib3.util.retry import Retry
|
|
7
|
+
from typing import Dict, Iterable, List, Optional, Union, cast
|
|
13
8
|
from urllib.parse import urlsplit
|
|
14
9
|
|
|
10
|
+
import requests
|
|
11
|
+
from requests.adapters import HTTPAdapter
|
|
12
|
+
from requests_cache import CachedSession
|
|
13
|
+
from requests_ratelimiter import LimiterAdapter
|
|
14
|
+
from urllib3.util.retry import Retry
|
|
15
|
+
|
|
15
16
|
from pori_python.types import ParsedVariant, PositionalVariant, Record
|
|
16
17
|
|
|
17
18
|
from .constants import DEFAULT_LIMIT, TYPES_TO_NOTATION, AA_3to1_MAPPING
|
|
18
19
|
|
|
19
|
-
QUERY_CACHE: Dict[Any, Any] = {}
|
|
20
|
-
|
|
21
20
|
# name the logger after the package to make it simple to disable for packages using this one as a dependency
|
|
22
21
|
# https://stackoverflow.com/questions/11029717/how-do-i-disable-log-messages-from-the-requests-library
|
|
23
22
|
|
|
24
23
|
logger = logging.getLogger('graphkb')
|
|
24
|
+
DEFAULT_LIMITER = LimiterAdapter(per_second=3)
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def convert_to_rid_list(records: Iterable[Record]) -> List[str]:
|
|
@@ -88,11 +88,17 @@ def millis_interval(start: datetime, end: datetime) -> int:
|
|
|
88
88
|
return millis
|
|
89
89
|
|
|
90
90
|
|
|
91
|
-
def
|
|
92
|
-
"""
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
def cache_filter(response):
|
|
92
|
+
"""
|
|
93
|
+
Only cache POST requests when they are for the /query endpoint. Otherwise they are creating content
|
|
94
|
+
and should not be cached
|
|
95
|
+
"""
|
|
96
|
+
request = response.request
|
|
97
|
+
|
|
98
|
+
if request.method == 'POST':
|
|
99
|
+
return request.url.rstrip('/').endswith('/query')
|
|
100
|
+
|
|
101
|
+
return True
|
|
96
102
|
|
|
97
103
|
|
|
98
104
|
class GraphKBConnection:
|
|
@@ -102,23 +108,89 @@ class GraphKBConnection:
|
|
|
102
108
|
username: str = '',
|
|
103
109
|
password: str = '',
|
|
104
110
|
use_global_cache: bool = True,
|
|
111
|
+
cache_name: str = '',
|
|
112
|
+
only_if_cached: bool = False,
|
|
113
|
+
session: Optional[requests.Session] = None,
|
|
114
|
+
limiter_kwargs: Optional[dict] = None,
|
|
115
|
+
**session_kwargs,
|
|
105
116
|
):
|
|
106
|
-
|
|
107
|
-
|
|
117
|
+
"""
|
|
118
|
+
Docstring for __init__
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
- use_global_cache: cache requests across all requests to GKB
|
|
122
|
+
- cache_name: Path or connection URL to the database which stors the requests cache. see https://requests-cache.readthedocs.io/en/v0.6.4/user_guide.html#cache-name
|
|
123
|
+
- only_if_cached: this will set the cache-control header for all requests to only-if-cached which will raise 504 errors if a request does not exist in the cache already rather than making a new network request
|
|
124
|
+
"""
|
|
125
|
+
session_cls = requests.Session
|
|
126
|
+
if session is not None:
|
|
127
|
+
if limiter_kwargs is not None:
|
|
128
|
+
raise NotImplementedError('cannot add limiter to an existing session')
|
|
129
|
+
if use_global_cache:
|
|
130
|
+
raise NotImplementedError(
|
|
131
|
+
'the use_global_cache parameter should not be used with a custom session'
|
|
132
|
+
)
|
|
133
|
+
if cache_name:
|
|
134
|
+
raise NotImplementedError(
|
|
135
|
+
'cache_name should not be used with a custom input session'
|
|
136
|
+
)
|
|
137
|
+
if not use_global_cache and cache_name:
|
|
138
|
+
raise NotImplementedError('cache_name only applies when use_global_cache is True')
|
|
139
|
+
if only_if_cached and not use_global_cache:
|
|
140
|
+
raise NotImplementedError(
|
|
141
|
+
'only_if_cached=True and use_global_cache=False are conflicting settings'
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
if use_global_cache:
|
|
145
|
+
session_cls = CachedSession
|
|
146
|
+
if not cache_name:
|
|
147
|
+
session_kwargs['backend'] = 'memory'
|
|
148
|
+
else:
|
|
149
|
+
session_kwargs['cache_name'] = cache_name
|
|
150
|
+
session_kwargs['allowable_methods'] = ['GET', 'POST']
|
|
151
|
+
session_kwargs['filter_fn'] = cache_filter
|
|
152
|
+
session_kwargs['cache_control'] = True
|
|
153
|
+
|
|
154
|
+
if 'PYTEST_CURRENT_TEST' in os.environ or only_if_cached:
|
|
155
|
+
if limiter_kwargs is not None:
|
|
156
|
+
logging.warning(
|
|
157
|
+
'rate limiting is by default turned off for tests and cache-only queries. Setting limiter to None'
|
|
158
|
+
)
|
|
159
|
+
limiter_kwargs = None
|
|
160
|
+
|
|
161
|
+
if not session:
|
|
162
|
+
self.http = session_cls(**session_kwargs)
|
|
163
|
+
else:
|
|
164
|
+
self.http = session
|
|
165
|
+
|
|
166
|
+
max_retries = Retry(
|
|
108
167
|
total=100,
|
|
109
168
|
connect=5,
|
|
110
169
|
status=5,
|
|
111
170
|
backoff_factor=5,
|
|
112
171
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
113
172
|
)
|
|
114
|
-
|
|
173
|
+
|
|
174
|
+
if limiter_kwargs:
|
|
175
|
+
adapter = LimiterAdapter(max_retries=max_retries, **limiter_kwargs)
|
|
176
|
+
else:
|
|
177
|
+
adapter = HTTPAdapter(max_retries=max_retries)
|
|
178
|
+
|
|
179
|
+
if not only_if_cached:
|
|
180
|
+
# do not need adapters at all for cache-only hits
|
|
181
|
+
self.http.mount('http://', adapter)
|
|
182
|
+
self.http.mount('https://', adapter)
|
|
183
|
+
self.only_if_cached = only_if_cached
|
|
184
|
+
|
|
115
185
|
self.token = ''
|
|
116
186
|
self.token_kc = ''
|
|
117
187
|
self.url = url
|
|
118
188
|
self.username = username
|
|
119
189
|
self.password = password
|
|
120
|
-
self.headers = {
|
|
121
|
-
|
|
190
|
+
self.headers = {
|
|
191
|
+
'Accept': 'application/json',
|
|
192
|
+
'Content-Type': 'application/json',
|
|
193
|
+
}
|
|
122
194
|
self.request_count = 0
|
|
123
195
|
self.first_request: Optional[datetime] = None
|
|
124
196
|
self.last_request: Optional[datetime] = None
|
|
@@ -137,7 +209,16 @@ class GraphKBConnection:
|
|
|
137
209
|
return self.request_count * 1000 / msec
|
|
138
210
|
return None
|
|
139
211
|
|
|
140
|
-
def request(
|
|
212
|
+
def request(
|
|
213
|
+
self,
|
|
214
|
+
endpoint: str,
|
|
215
|
+
method: str = 'GET',
|
|
216
|
+
headers: Optional[dict[str, str]] = None,
|
|
217
|
+
ignore_cache=False,
|
|
218
|
+
force_refresh=False,
|
|
219
|
+
only_if_cached=False,
|
|
220
|
+
**kwargs,
|
|
221
|
+
) -> Dict:
|
|
141
222
|
"""Request wrapper to handle adding common headers and logging.
|
|
142
223
|
|
|
143
224
|
Args:
|
|
@@ -147,6 +228,13 @@ class GraphKBConnection:
|
|
|
147
228
|
Returns:
|
|
148
229
|
dict: the json response as a python dict
|
|
149
230
|
"""
|
|
231
|
+
if headers is None:
|
|
232
|
+
headers = {}
|
|
233
|
+
|
|
234
|
+
if ignore_cache or force_refresh:
|
|
235
|
+
headers['Cache-Control'] = 'no-cache'
|
|
236
|
+
elif only_if_cached or self.only_if_cached:
|
|
237
|
+
headers['Cache-Control'] = 'only-if-cached'
|
|
150
238
|
url = join_url(self.url, endpoint)
|
|
151
239
|
self.request_count += 1
|
|
152
240
|
connect_timeout = 7
|
|
@@ -158,6 +246,11 @@ class GraphKBConnection:
|
|
|
158
246
|
if endpoint in ['query', 'parse']:
|
|
159
247
|
timeout = (connect_timeout, read_timeout)
|
|
160
248
|
|
|
249
|
+
request_headers = {}
|
|
250
|
+
request_headers.update(self.headers)
|
|
251
|
+
if headers is not None:
|
|
252
|
+
request_headers.update(headers)
|
|
253
|
+
|
|
161
254
|
start_time = datetime.now()
|
|
162
255
|
|
|
163
256
|
if not self.first_request:
|
|
@@ -176,11 +269,12 @@ class GraphKBConnection:
|
|
|
176
269
|
try:
|
|
177
270
|
if need_refresh_login:
|
|
178
271
|
self.refresh_login()
|
|
272
|
+
request_headers.update(self.headers)
|
|
179
273
|
need_refresh_login = False
|
|
180
274
|
|
|
181
275
|
self.request_count += 1
|
|
182
|
-
resp =
|
|
183
|
-
method, url, headers=
|
|
276
|
+
resp = self.http.request(
|
|
277
|
+
method, url, headers=request_headers, timeout=timeout, **kwargs
|
|
184
278
|
)
|
|
185
279
|
if resp.status_code == 401 or resp.status_code == 403:
|
|
186
280
|
logger.debug(f'/{endpoint} - {resp.status_code} - retrying')
|
|
@@ -293,39 +387,29 @@ class GraphKBConnection:
|
|
|
293
387
|
def refresh_login(self) -> None:
|
|
294
388
|
self.login(self.username, self.password)
|
|
295
389
|
|
|
296
|
-
def set_cache_data(self, request_body: Dict, result: List[Record]) -> None:
|
|
297
|
-
"""Explicitly add a query to the cache."""
|
|
298
|
-
hash_code = cache_key(request_body)
|
|
299
|
-
self.cache[hash_code] = result
|
|
300
|
-
|
|
301
390
|
def query(
|
|
302
391
|
self,
|
|
303
392
|
request_body: Dict = {},
|
|
304
393
|
paginate: bool = True,
|
|
305
|
-
ignore_cache: bool = False,
|
|
306
|
-
force_refresh: bool = False,
|
|
307
394
|
limit: int = DEFAULT_LIMIT,
|
|
395
|
+
**kwargs,
|
|
308
396
|
) -> List[Record]:
|
|
309
397
|
"""
|
|
310
398
|
Query GraphKB
|
|
311
399
|
"""
|
|
312
|
-
result: List[Record] = []
|
|
313
|
-
hash_code = ''
|
|
314
|
-
|
|
315
|
-
if not ignore_cache and paginate:
|
|
316
|
-
hash_code = cache_key(request_body)
|
|
317
|
-
if hash_code in self.cache and not force_refresh:
|
|
318
|
-
return self.cache[hash_code]
|
|
319
400
|
|
|
401
|
+
result: List[Record] = []
|
|
320
402
|
while True:
|
|
321
|
-
content = self.post(
|
|
403
|
+
content = self.post(
|
|
404
|
+
'query',
|
|
405
|
+
data={**request_body, 'limit': limit, 'skip': len(result)},
|
|
406
|
+
**kwargs,
|
|
407
|
+
)
|
|
322
408
|
records = content['result']
|
|
323
409
|
result.extend(records)
|
|
324
410
|
if len(records) < limit or not paginate:
|
|
325
411
|
break
|
|
326
412
|
|
|
327
|
-
if not ignore_cache and paginate:
|
|
328
|
-
self.cache[hash_code] = result
|
|
329
413
|
return result
|
|
330
414
|
|
|
331
415
|
def parse(self, hgvs_string: str, requireFeatures: bool = False) -> ParsedVariant:
|
|
@@ -500,7 +584,9 @@ def stripDisplayName(displayName: str, withRef: bool = True, withRefSeq: bool =
|
|
|
500
584
|
|
|
501
585
|
|
|
502
586
|
def stringifyVariant(
|
|
503
|
-
variant: Union[PositionalVariant, ParsedVariant],
|
|
587
|
+
variant: Union[PositionalVariant, ParsedVariant],
|
|
588
|
+
withRef: bool = True,
|
|
589
|
+
withRefSeq: bool = True,
|
|
504
590
|
) -> str:
|
|
505
591
|
"""
|
|
506
592
|
Convert variant record to a string representation (displayName/hgvs)
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
from typing import Callable, Dict, Iterable, List, Set,
|
|
1
|
+
from typing import Callable, Dict, Iterable, List, Set, Union, cast
|
|
2
2
|
|
|
3
3
|
from pori_python.types import Ontology
|
|
4
4
|
|
|
5
5
|
from . import GraphKBConnection
|
|
6
6
|
from .util import convert_to_rid_list
|
|
7
7
|
|
|
8
|
+
DEFAULT_RETURN_PROPERTIES = ['sourceId', 'sourceIdVersion', 'deprecated', 'name', '@rid']
|
|
9
|
+
|
|
8
10
|
|
|
9
11
|
def query_by_name(ontology_class: str, base_term_name: Union[str, list[str]]) -> Dict:
|
|
10
12
|
return {'target': ontology_class, 'filters': {'name': base_term_name}}
|
|
@@ -17,6 +19,7 @@ def get_equivalent_terms(
|
|
|
17
19
|
ontology_class: str = 'Vocabulary',
|
|
18
20
|
ignore_cache: bool = False,
|
|
19
21
|
build_base_query: Callable = query_by_name,
|
|
22
|
+
return_properties=DEFAULT_RETURN_PROPERTIES,
|
|
20
23
|
) -> List[Ontology]:
|
|
21
24
|
"""
|
|
22
25
|
Get a list of terms equivalent to the current term up to the root term
|
|
@@ -35,7 +38,7 @@ def get_equivalent_terms(
|
|
|
35
38
|
'target': {'target': base_records, 'queryType': 'descendants'},
|
|
36
39
|
'queryType': 'similarTo',
|
|
37
40
|
'treeEdges': [],
|
|
38
|
-
'returnProperties':
|
|
41
|
+
'returnProperties': return_properties,
|
|
39
42
|
},
|
|
40
43
|
ignore_cache=ignore_cache,
|
|
41
44
|
),
|
|
@@ -54,13 +57,7 @@ def get_equivalent_terms(
|
|
|
54
57
|
'target': {'target': root_records, 'queryType': 'descendants'},
|
|
55
58
|
'queryType': 'similarTo',
|
|
56
59
|
'treeEdges': [],
|
|
57
|
-
'returnProperties': [
|
|
58
|
-
'sourceId',
|
|
59
|
-
'sourceIdVersion',
|
|
60
|
-
'deprecated',
|
|
61
|
-
'name',
|
|
62
|
-
'@rid',
|
|
63
|
-
],
|
|
60
|
+
'returnProperties': ['@rid'],
|
|
64
61
|
},
|
|
65
62
|
ignore_cache=ignore_cache,
|
|
66
63
|
)
|
|
@@ -78,6 +75,7 @@ def get_term_tree(
|
|
|
78
75
|
include_superclasses: bool = True,
|
|
79
76
|
ignore_cache: bool = False,
|
|
80
77
|
build_base_query: Callable = query_by_name,
|
|
78
|
+
return_properties=DEFAULT_RETURN_PROPERTIES,
|
|
81
79
|
) -> List[Ontology]:
|
|
82
80
|
"""
|
|
83
81
|
Get terms equivalent to the base term by traversing the subclassOf tree and expanding related
|
|
@@ -105,7 +103,7 @@ def get_term_tree(
|
|
|
105
103
|
'target': {'target': base_records, 'queryType': 'ancestors'},
|
|
106
104
|
'queryType': 'similarTo',
|
|
107
105
|
'treeEdges': [],
|
|
108
|
-
'returnProperties':
|
|
106
|
+
'returnProperties': return_properties,
|
|
109
107
|
},
|
|
110
108
|
ignore_cache=ignore_cache,
|
|
111
109
|
),
|
|
@@ -119,6 +117,7 @@ def get_term_tree(
|
|
|
119
117
|
ontology_class=ontology_class,
|
|
120
118
|
ignore_cache=ignore_cache,
|
|
121
119
|
build_base_query=build_base_query,
|
|
120
|
+
return_properties=return_properties,
|
|
122
121
|
)
|
|
123
122
|
else:
|
|
124
123
|
parent_terms = []
|
|
@@ -181,9 +180,6 @@ def get_terms_set(
|
|
|
181
180
|
) -> Set[str]:
|
|
182
181
|
"""Get a set of vocabulary rids given some base/parent term names."""
|
|
183
182
|
base_terms = [base_terms] if isinstance(base_terms, str) else base_terms
|
|
184
|
-
cache_key = tuple(sorted(base_terms))
|
|
185
|
-
if graphkb_conn.cache.get(cache_key, None) and not ignore_cache:
|
|
186
|
-
return graphkb_conn.cache[cache_key]
|
|
187
183
|
terms = set()
|
|
188
184
|
for base_term in base_terms:
|
|
189
185
|
terms.update(
|
|
@@ -193,6 +189,4 @@ def get_terms_set(
|
|
|
193
189
|
)
|
|
194
190
|
)
|
|
195
191
|
)
|
|
196
|
-
if not ignore_cache:
|
|
197
|
-
graphkb_conn.cache[cache_key] = terms
|
|
198
192
|
return terms
|
|
@@ -322,15 +322,26 @@ def create_key_alterations(
|
|
|
322
322
|
|
|
323
323
|
counts[type_mapping[variant_type]].add(variant_key)
|
|
324
324
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
325
|
+
# DEVSU-2013 returning an alteration object for IPR API to look up an observed variant for association at report creation.
|
|
326
|
+
# variantType: the type of variant (mut, cnv, sv, exp)
|
|
327
|
+
# variant: the observed variant's key property (eg: e90293983749cf4)
|
|
328
|
+
# Example value: {'variantType': 'mut', 'variant': 'e90293983749cf4'}
|
|
329
|
+
if kb_match['category'] in GERMLINE_BASE_TERMS:
|
|
330
|
+
# only include germline-category matches when the observed variant is germline
|
|
331
|
+
if variant.get('germline'):
|
|
332
|
+
alterations.append(
|
|
333
|
+
{
|
|
334
|
+
'variantType': variant_type,
|
|
335
|
+
'variant': variant['key'],
|
|
336
|
+
},
|
|
337
|
+
)
|
|
332
338
|
else:
|
|
333
|
-
alterations.append(
|
|
339
|
+
alterations.append(
|
|
340
|
+
{
|
|
341
|
+
'variantType': variant_type,
|
|
342
|
+
'variant': variant['key'],
|
|
343
|
+
},
|
|
344
|
+
)
|
|
334
345
|
|
|
335
346
|
counted_variants = set.union(*counts.values())
|
|
336
347
|
counts['variantsUnknown'] = set()
|
|
@@ -340,8 +351,17 @@ def create_key_alterations(
|
|
|
340
351
|
if variant['variant'] and variant['key'] not in counted_variants:
|
|
341
352
|
counts['variantsUnknown'].add(variant['key'])
|
|
342
353
|
|
|
354
|
+
unique_alterations = []
|
|
355
|
+
seen = set()
|
|
356
|
+
for alt in alterations:
|
|
357
|
+
dedupe_key = (alt['variantType'], alt['variant'])
|
|
358
|
+
if dedupe_key in seen:
|
|
359
|
+
continue
|
|
360
|
+
seen.add(dedupe_key)
|
|
361
|
+
unique_alterations.append(alt)
|
|
362
|
+
|
|
343
363
|
return (
|
|
344
|
-
|
|
364
|
+
unique_alterations,
|
|
345
365
|
{k: len(v) for k, v in counts.items()},
|
|
346
366
|
)
|
|
347
367
|
|
|
@@ -3,12 +3,13 @@ from __future__ import annotations
|
|
|
3
3
|
import argparse
|
|
4
4
|
import datetime
|
|
5
5
|
import json
|
|
6
|
-
import jsonschema.exceptions
|
|
7
6
|
import logging
|
|
8
7
|
import os
|
|
9
|
-
import pandas as pd
|
|
10
8
|
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
|
11
|
-
from
|
|
9
|
+
from collections.abc import Callable, Sequence
|
|
10
|
+
|
|
11
|
+
import jsonschema.exceptions
|
|
12
|
+
import pandas as pd
|
|
12
13
|
|
|
13
14
|
from pori_python.graphkb import GraphKBConnection
|
|
14
15
|
from pori_python.graphkb.genes import get_gene_information
|
|
@@ -33,8 +34,8 @@ from .inputs import (
|
|
|
33
34
|
preprocess_cosmic,
|
|
34
35
|
preprocess_expression_variants,
|
|
35
36
|
preprocess_hla,
|
|
36
|
-
preprocess_msi,
|
|
37
37
|
preprocess_hrd,
|
|
38
|
+
preprocess_msi,
|
|
38
39
|
preprocess_signature_variants,
|
|
39
40
|
preprocess_small_mutations,
|
|
40
41
|
preprocess_structural_variants,
|
|
@@ -42,14 +43,14 @@ from .inputs import (
|
|
|
42
43
|
validate_report_content,
|
|
43
44
|
)
|
|
44
45
|
from .ipr import (
|
|
46
|
+
add_transcript_flags,
|
|
45
47
|
create_key_alterations,
|
|
46
48
|
filter_structural_variants,
|
|
47
49
|
germline_kb_matches,
|
|
48
50
|
get_kb_disease_matches,
|
|
49
51
|
get_kb_matches_sections,
|
|
50
|
-
select_expression_plots,
|
|
51
52
|
get_variant_flags,
|
|
52
|
-
|
|
53
|
+
select_expression_plots,
|
|
53
54
|
)
|
|
54
55
|
from .summary import auto_analyst_comments, get_ipr_analyst_comments
|
|
55
56
|
from .therapeutic_options import create_therapeutic_options
|
|
@@ -65,7 +66,7 @@ RENAMED_GENE_PROPERTIES = {
|
|
|
65
66
|
|
|
66
67
|
def file_path(path: str) -> str:
|
|
67
68
|
if not os.path.exists(path):
|
|
68
|
-
raise argparse.ArgumentTypeError(f'{
|
|
69
|
+
raise argparse.ArgumentTypeError(f'{path!r} is not a valid filename. does not exist')
|
|
69
70
|
return path
|
|
70
71
|
|
|
71
72
|
|
|
@@ -215,7 +216,7 @@ def command_interface() -> None:
|
|
|
215
216
|
)
|
|
216
217
|
|
|
217
218
|
|
|
218
|
-
def clean_unsupported_content(upload_content:
|
|
219
|
+
def clean_unsupported_content(upload_content: dict, ipr_spec: dict = {}) -> dict:
|
|
219
220
|
"""Remove unsupported content.
|
|
220
221
|
This content is either added to facilitate creation
|
|
221
222
|
or to support upcoming and soon to be supported content that we would like
|
|
@@ -223,7 +224,7 @@ def clean_unsupported_content(upload_content: Dict, ipr_spec: Dict = {}) -> Dict
|
|
|
223
224
|
"""
|
|
224
225
|
if (
|
|
225
226
|
ipr_spec
|
|
226
|
-
and 'components' in ipr_spec
|
|
227
|
+
and 'components' in ipr_spec
|
|
227
228
|
and 'schemas' in ipr_spec['components'].keys()
|
|
228
229
|
and 'genesCreate' in ipr_spec['components']['schemas'].keys()
|
|
229
230
|
and 'properties' in ipr_spec['components']['schemas']['genesCreate'].keys()
|
|
@@ -253,7 +254,7 @@ def clean_unsupported_content(upload_content: Dict, ipr_spec: Dict = {}) -> Dict
|
|
|
253
254
|
)
|
|
254
255
|
|
|
255
256
|
# remove any unhandled incompatible keys
|
|
256
|
-
removed_keys:
|
|
257
|
+
removed_keys: dict[str, int] = {}
|
|
257
258
|
for gene in upload_content['genes']:
|
|
258
259
|
unsupported_keys = [key for key in gene.keys() if key not in genes_spec]
|
|
259
260
|
for key in unsupported_keys:
|
|
@@ -315,7 +316,7 @@ def clean_unsupported_content(upload_content: Dict, ipr_spec: Dict = {}) -> Dict
|
|
|
315
316
|
return upload_content
|
|
316
317
|
|
|
317
318
|
|
|
318
|
-
def create_report(**kwargs) ->
|
|
319
|
+
def create_report(**kwargs) -> dict:
|
|
319
320
|
logger.warning("Deprecated function 'create_report' called - use ipr_report instead")
|
|
320
321
|
return ipr_report(**kwargs)
|
|
321
322
|
|
|
@@ -323,7 +324,7 @@ def create_report(**kwargs) -> Dict:
|
|
|
323
324
|
def ipr_report(
|
|
324
325
|
username: str,
|
|
325
326
|
password: str,
|
|
326
|
-
content:
|
|
327
|
+
content: dict,
|
|
327
328
|
ipr_url: str = '',
|
|
328
329
|
log_level: str = 'info',
|
|
329
330
|
output_json_path: str = '',
|
|
@@ -336,7 +337,7 @@ def ipr_report(
|
|
|
336
337
|
generate_therapeutics: bool = False,
|
|
337
338
|
generate_comments: bool = True,
|
|
338
339
|
match_germline: bool = False,
|
|
339
|
-
custom_kb_match_filter:
|
|
340
|
+
custom_kb_match_filter: Callable | None = None,
|
|
340
341
|
async_upload: bool = False,
|
|
341
342
|
mins_to_wait: int = 10,
|
|
342
343
|
include_ipr_variant_text: bool = True,
|
|
@@ -349,7 +350,7 @@ def ipr_report(
|
|
|
349
350
|
ignore_extra_fields: bool = False,
|
|
350
351
|
tmb_high: float = TMB_SIGNATURE_HIGH_THRESHOLD,
|
|
351
352
|
transcript_flags: str = '',
|
|
352
|
-
) ->
|
|
353
|
+
) -> dict:
|
|
353
354
|
"""Run the matching and create the report JSON for upload to IPR.
|
|
354
355
|
|
|
355
356
|
Args:
|
|
@@ -431,7 +432,7 @@ def ipr_report(
|
|
|
431
432
|
transcript_flags_df = load_transcript_flags(transcript_flags)
|
|
432
433
|
|
|
433
434
|
# INPUT VARIANTS VALIDATION & PREPROCESSING (OBSERVED BIOMARKERS)
|
|
434
|
-
signature_variants:
|
|
435
|
+
signature_variants: list[IprSignatureVariant] = preprocess_signature_variants(
|
|
435
436
|
[
|
|
436
437
|
*preprocess_cosmic(content.get('cosmicSignatures', [])), # includes dMMR
|
|
437
438
|
*preprocess_hla(content.get('hlaTypes', [])),
|
|
@@ -444,14 +445,14 @@ def ipr_report(
|
|
|
444
445
|
*preprocess_hrd(content.get('hrd', None)),
|
|
445
446
|
]
|
|
446
447
|
)
|
|
447
|
-
small_mutations:
|
|
448
|
+
small_mutations: list[IprSmallMutationVariant] = preprocess_small_mutations(
|
|
448
449
|
content.get('smallMutations', [])
|
|
449
450
|
)
|
|
450
|
-
structural_variants:
|
|
451
|
+
structural_variants: list[IprFusionVariant] = preprocess_structural_variants(
|
|
451
452
|
content.get('structuralVariants', [])
|
|
452
453
|
)
|
|
453
|
-
copy_variants:
|
|
454
|
-
expression_variants:
|
|
454
|
+
copy_variants: list[IprCopyVariant] = preprocess_copy_variants(content.get('copyVariants', []))
|
|
455
|
+
expression_variants: list[IprExprVariant] = preprocess_expression_variants(
|
|
455
456
|
content.get('expressionVariants', [])
|
|
456
457
|
)
|
|
457
458
|
|
|
@@ -459,7 +460,7 @@ def ipr_report(
|
|
|
459
460
|
if expression_variants:
|
|
460
461
|
check_comparators(content, expression_variants)
|
|
461
462
|
|
|
462
|
-
genes_with_variants:
|
|
463
|
+
genes_with_variants: set[str] = check_variant_links(
|
|
463
464
|
small_mutations, expression_variants, copy_variants, structural_variants
|
|
464
465
|
)
|
|
465
466
|
|
|
@@ -478,12 +479,12 @@ def ipr_report(
|
|
|
478
479
|
|
|
479
480
|
# Matching disease RIDs from GraphKB using term tree
|
|
480
481
|
# (Will raise uncatched error if no match)
|
|
481
|
-
disease_match_records: list[
|
|
482
|
+
disease_match_records: list[dict] = get_kb_disease_matches(graphkb_conn, kb_disease_match)
|
|
482
483
|
disease_match_rids: list[str] = [item['@rid'] for item in disease_match_records]
|
|
483
484
|
disease_match_names: list[str] = [item['name'] for item in disease_match_records]
|
|
484
485
|
|
|
485
486
|
# GKB MATCHING (AKA ANNOTATION)
|
|
486
|
-
gkb_matches:
|
|
487
|
+
gkb_matches: list[Hashabledict] = annotate_variants(
|
|
487
488
|
graphkb_conn=graphkb_conn,
|
|
488
489
|
interactive=interactive,
|
|
489
490
|
disease_matches=disease_match_rids,
|
|
@@ -654,7 +655,16 @@ def ipr_report(
|
|
|
654
655
|
output, mins_to_wait, async_upload, ignore_extra_fields
|
|
655
656
|
)
|
|
656
657
|
logger.info(ipr_result)
|
|
657
|
-
|
|
658
|
+
if async_upload and 'report' in ipr_result:
|
|
659
|
+
# async uploads return the report nested under 'report' alongside
|
|
660
|
+
# job-tracking fields (eg. 'ident' of the job, 'jobStatus')
|
|
661
|
+
# full contents of 'report' aren't needed and can be ignored.
|
|
662
|
+
# no 'message' value is available for async results
|
|
663
|
+
# but 'ident' is, and is required by users.
|
|
664
|
+
output['ident'] = ipr_result['report']['ident']
|
|
665
|
+
else:
|
|
666
|
+
# for sync uploads, add 'ident' and 'message' fields to the output.
|
|
667
|
+
output.update(ipr_result)
|
|
658
668
|
except Exception as err:
|
|
659
669
|
upload_error = err
|
|
660
670
|
logger.error(f'ipr_conn.upload_report failed: {err}', exc_info=True)
|