pori-python 1.4.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pori_python-1.4.0/pori_python.egg-info → pori_python-1.6.0}/PKG-INFO +4 -1
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/constants.py +4 -1
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/genes.py +130 -12
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/match.py +2 -27
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/util.py +193 -39
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/vocab.py +10 -16
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/connection.py +45 -14
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/content.spec.json +147 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/inputs.py +58 -1
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/ipr.py +109 -10
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/main.py +105 -25
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/summary.py +6 -6
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/util.py +5 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/types.py +2 -1
- {pori_python-1.4.0 → pori_python-1.6.0/pori_python.egg-info}/PKG-INFO +4 -1
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python.egg-info/SOURCES.txt +1 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python.egg-info/requires.txt +3 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/setup.cfg +8 -1
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/test_genes.py +33 -8
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/test_match.py +0 -16
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/test_statement.py +14 -3
- pori_python-1.6.0/tests/test_graphkb/test_util.py +393 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_annotate.py +25 -1
- pori_python-1.6.0/tests/test_ipr/test_connection.py +219 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_inputs.py +160 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_ipr.py +184 -0
- pori_python-1.6.0/tests/test_ipr/test_ipr_integration.py +102 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_main.py +30 -2
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_probe.py +5 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_summary.py +32 -32
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/test_upload.py +73 -5
- pori_python-1.6.0/tests/test_ipr/test_util.py +48 -0
- pori_python-1.4.0/tests/test_graphkb/test_util.py +0 -151
- pori_python-1.4.0/tests/test_ipr/test_connection.py +0 -97
- pori_python-1.4.0/tests/test_ipr/test_util.py +0 -29
- {pori_python-1.4.0 → pori_python-1.6.0}/LICENSE +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/MANIFEST.in +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/README.md +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/graphkb/statement.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/annotate.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/constants.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/ipr/therapeutic_options.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python/py.typed +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python.egg-info/dependency_links.txt +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python.egg-info/entry_points.txt +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pori_python.egg-info/top_level.txt +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/pyproject.toml +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/setup.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/data.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/test_graphkb.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_graphkb/test_vocab.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/__init__.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/constants.py +0 -0
- {pori_python-1.4.0 → pori_python-1.6.0}/tests/test_ipr/util.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pori_python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Home-page: https://github.com/bcgsc/pori_python
|
|
5
5
|
Author-email: dat@bcgsc.ca
|
|
6
6
|
Maintainer-email: dat@bcgsc.ca
|
|
@@ -12,6 +12,9 @@ Requires-Dist: pandas>=1.1.0
|
|
|
12
12
|
Requires-Dist: requests
|
|
13
13
|
Requires-Dist: tqdm
|
|
14
14
|
Requires-Dist: typing_extensions<5,>=3.7.4.2
|
|
15
|
+
Requires-Dist: requests-cache<2.0,>=1.3.0
|
|
16
|
+
Requires-Dist: requests-ratelimiter
|
|
17
|
+
Requires-Dist: pyrate-limiter
|
|
15
18
|
Provides-Extra: deploy
|
|
16
19
|
Requires-Dist: twine; extra == "deploy"
|
|
17
20
|
Requires-Dist: wheel; extra == "deploy"
|
|
@@ -59,7 +59,10 @@ ONCOKB_SOURCE_NAME = 'oncokb'
|
|
|
59
59
|
TSO500_SOURCE_NAME = 'tso500'
|
|
60
60
|
ONCOGENE = 'oncogenic'
|
|
61
61
|
TUMOUR_SUPPRESSIVE = 'tumour suppressive'
|
|
62
|
-
CANCER_GENE =
|
|
62
|
+
CANCER_GENE = [
|
|
63
|
+
'cancer gene',
|
|
64
|
+
'tumourigenesis',
|
|
65
|
+
] # KBDEV-1532. tumourigenesis for backward compatibility
|
|
63
66
|
FUSION_NAMES = ['structural variant', 'fusion']
|
|
64
67
|
|
|
65
68
|
GSC_PHARMACOGENOMIC_SOURCE_EXCLUDE_LIST = ['cancer genome interpreter', 'civic']
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Any, Dict, List, Sequence, Set, Tuple, cast
|
|
5
|
+
from typing import Any, Dict, List, Sequence, Set, Tuple, cast, Union
|
|
6
6
|
from typing_extensions import deprecated
|
|
7
7
|
|
|
8
8
|
from pori_python.types import IprGene, Ontology, Record, Statement, Variant
|
|
@@ -27,8 +27,117 @@ from .util import get_rid, logger, looks_like_rid
|
|
|
27
27
|
from .vocab import get_terms_set
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
def get_cancer_gene_flags(
|
|
31
|
+
conn: GraphKBConnection,
|
|
32
|
+
flags: bool = False,
|
|
33
|
+
ignore_cache: bool = False,
|
|
34
|
+
) -> Union[List[Record], Dict[str, List[Record]]]:
|
|
35
|
+
"""
|
|
36
|
+
Return all cancer genes, optionally sorted by flags.
|
|
37
|
+
|
|
38
|
+
Flag definitions:
|
|
39
|
+
oncogenic: relevance 'oncogenic' from OncoKB
|
|
40
|
+
tumourSuppressive: relevance 'tumour suppressive' from OncoKB
|
|
41
|
+
cancerGene: relevance 'cancer gene' AND child terms ('oncogenic', 'tumour suppressive', 'other cancer gene'), from OncoKB AND TSO500
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
conn: the graphkb connection object
|
|
45
|
+
flags: if the results should be sorted by flags
|
|
46
|
+
ignore_cache: if cache should be ignored when querying GraphKB API
|
|
47
|
+
|
|
48
|
+
Returns (if flags=False; default): list of unique gene records
|
|
49
|
+
[ <record>, <record>, ... ]
|
|
50
|
+
|
|
51
|
+
Returns (if flags=True): dict of flags as keys, and list of gene records as value
|
|
52
|
+
{
|
|
53
|
+
'oncogenic': [ <record>, <record>, ... ],
|
|
54
|
+
'tumourSuppressive': [ <record>, <record>, ... ],
|
|
55
|
+
'cancerGene': [ <record>, <record>, ... ],
|
|
56
|
+
}
|
|
57
|
+
"""
|
|
58
|
+
# all cancer gene statements
|
|
59
|
+
cancer_genes = conn.get_related_terms(
|
|
60
|
+
terms=CANCER_GENE,
|
|
61
|
+
subgraphType='children',
|
|
62
|
+
)
|
|
63
|
+
statements = cast(
|
|
64
|
+
List[Statement],
|
|
65
|
+
conn.query(
|
|
66
|
+
{
|
|
67
|
+
'target': 'Statement',
|
|
68
|
+
'filters': {
|
|
69
|
+
'relevance': {'target': 'Vocabulary', 'filters': {'name': cancer_genes}}
|
|
70
|
+
},
|
|
71
|
+
'returnProperties': [
|
|
72
|
+
'source.name',
|
|
73
|
+
'relevance.name',
|
|
74
|
+
*[f'subject.{prop}' for prop in GENE_RETURN_PROPERTIES],
|
|
75
|
+
],
|
|
76
|
+
},
|
|
77
|
+
ignore_cache=ignore_cache,
|
|
78
|
+
),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# post-query filtering (faster)
|
|
82
|
+
cancerGeneStms = list(
|
|
83
|
+
filter(
|
|
84
|
+
lambda r: (
|
|
85
|
+
r['subject']['@class'] == 'Feature'
|
|
86
|
+
and r['subject']['biotype'] == 'gene'
|
|
87
|
+
and r['source']['name'] in [ONCOKB_SOURCE_NAME, TSO500_SOURCE_NAME]
|
|
88
|
+
),
|
|
89
|
+
statements,
|
|
90
|
+
)
|
|
91
|
+
)
|
|
92
|
+
oncogenicStms = list(
|
|
93
|
+
filter(
|
|
94
|
+
lambda r: (
|
|
95
|
+
r['relevance']['name'] == ONCOGENE and r['source']['name'] == ONCOKB_SOURCE_NAME
|
|
96
|
+
),
|
|
97
|
+
cancerGeneStms,
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
tumourSuppressiveStms = list(
|
|
101
|
+
filter(
|
|
102
|
+
lambda r: (
|
|
103
|
+
r['relevance']['name'] == TUMOUR_SUPPRESSIVE
|
|
104
|
+
and r['source']['name'] == ONCOKB_SOURCE_NAME
|
|
105
|
+
),
|
|
106
|
+
cancerGeneStms,
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Returning a sorted list of unique gene records, based on iProbe requirements
|
|
111
|
+
# Unique by name, sorted by displayName
|
|
112
|
+
if not flags:
|
|
113
|
+
seen: set = set()
|
|
114
|
+
unique_genes: List[Record] = []
|
|
115
|
+
for r in cancerGeneStms:
|
|
116
|
+
name = r['subject']['name']
|
|
117
|
+
if name not in seen:
|
|
118
|
+
seen.add(name)
|
|
119
|
+
unique_genes.append(r['subject'])
|
|
120
|
+
|
|
121
|
+
return cast(
|
|
122
|
+
List[Record],
|
|
123
|
+
sorted(unique_genes, key=lambda gene: gene['displayName']),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# Returning a Dict of flags, with list of associated gene records
|
|
127
|
+
# Duplicates are ok
|
|
128
|
+
return {
|
|
129
|
+
'cancerGene': [r['subject'] for r in cancerGeneStms],
|
|
130
|
+
'oncogenic': [r['subject'] for r in oncogenicStms],
|
|
131
|
+
'tumourSuppressive': [r['subject'] for r in tumourSuppressiveStms],
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@deprecated('functionality replaced by get_cancer_gene_flags')
|
|
30
136
|
def _get_tumourigenesis_genes_list(
|
|
31
|
-
conn: GraphKBConnection,
|
|
137
|
+
conn: GraphKBConnection,
|
|
138
|
+
relevance: Union[str, List[str]],
|
|
139
|
+
sources: Union[str, List[str]],
|
|
140
|
+
ignore_cache: bool = False,
|
|
32
141
|
) -> List[Ontology]:
|
|
33
142
|
statements = cast(
|
|
34
143
|
List[Statement],
|
|
@@ -57,6 +166,7 @@ def _get_tumourigenesis_genes_list(
|
|
|
57
166
|
return [gene for gene in genes.values()]
|
|
58
167
|
|
|
59
168
|
|
|
169
|
+
@deprecated('functionality replaced by get_cancer_gene_flags')
|
|
60
170
|
def get_oncokb_oncogenes(conn: GraphKBConnection) -> List[Ontology]:
|
|
61
171
|
"""Get the list of oncogenes stored in GraphKB derived from OncoKB.
|
|
62
172
|
|
|
@@ -66,9 +176,10 @@ def get_oncokb_oncogenes(conn: GraphKBConnection) -> List[Ontology]:
|
|
|
66
176
|
Returns:
|
|
67
177
|
gene (Feature) records
|
|
68
178
|
"""
|
|
69
|
-
return _get_tumourigenesis_genes_list(conn, ONCOGENE,
|
|
179
|
+
return _get_tumourigenesis_genes_list(conn, ONCOGENE, ONCOKB_SOURCE_NAME)
|
|
70
180
|
|
|
71
181
|
|
|
182
|
+
@deprecated('functionality replaced by get_cancer_gene_flags')
|
|
72
183
|
def get_oncokb_tumour_supressors(conn: GraphKBConnection) -> List[Ontology]:
|
|
73
184
|
"""Get the list of tumour supressor genes stored in GraphKB derived from OncoKB.
|
|
74
185
|
|
|
@@ -78,11 +189,14 @@ def get_oncokb_tumour_supressors(conn: GraphKBConnection) -> List[Ontology]:
|
|
|
78
189
|
Returns:
|
|
79
190
|
gene (Feature) records
|
|
80
191
|
"""
|
|
81
|
-
return _get_tumourigenesis_genes_list(conn, TUMOUR_SUPPRESSIVE,
|
|
192
|
+
return _get_tumourigenesis_genes_list(conn, TUMOUR_SUPPRESSIVE, ONCOKB_SOURCE_NAME)
|
|
82
193
|
|
|
83
194
|
|
|
195
|
+
@deprecated('functionality replaced by get_cancer_gene_flags')
|
|
84
196
|
def get_cancer_genes(conn: GraphKBConnection) -> List[Ontology]:
|
|
85
|
-
"""
|
|
197
|
+
"""
|
|
198
|
+
Get the list of cancer genes stored in GraphKB derived from OncoKB & TSO500.
|
|
199
|
+
Cancer genes include oncogenes, tumour supressor genes and other cancer genes.
|
|
86
200
|
|
|
87
201
|
Args:
|
|
88
202
|
conn: the graphkb connection object
|
|
@@ -90,8 +204,12 @@ def get_cancer_genes(conn: GraphKBConnection) -> List[Ontology]:
|
|
|
90
204
|
Returns:
|
|
91
205
|
gene (Feature) records
|
|
92
206
|
"""
|
|
207
|
+
cancer_gene_terms = conn.get_related_terms(
|
|
208
|
+
terms=CANCER_GENE,
|
|
209
|
+
subgraphType='children',
|
|
210
|
+
)
|
|
93
211
|
return _get_tumourigenesis_genes_list(
|
|
94
|
-
conn,
|
|
212
|
+
conn, cancer_gene_terms, [ONCOKB_SOURCE_NAME, TSO500_SOURCE_NAME]
|
|
95
213
|
)
|
|
96
214
|
|
|
97
215
|
|
|
@@ -513,12 +631,12 @@ def get_gene_information(
|
|
|
513
631
|
# PositionalVariant without a reference2 implies a smallMutation type
|
|
514
632
|
gene_flags['knownSmallMutation'].add(condition['reference1']) # type: ignore
|
|
515
633
|
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
gene_flags['
|
|
520
|
-
|
|
521
|
-
gene_flags['cancerGeneListMatch'] = convert_to_rid_set(
|
|
634
|
+
# cancer gene flags
|
|
635
|
+
logger.info('fetching cancer genes')
|
|
636
|
+
cancer_gene_flags = get_cancer_gene_flags(graphkb_conn, flags=True)
|
|
637
|
+
gene_flags['oncogene'] = convert_to_rid_set(cancer_gene_flags['oncogenic'])
|
|
638
|
+
gene_flags['tumourSuppressor'] = convert_to_rid_set(cancer_gene_flags['tumourSuppressive'])
|
|
639
|
+
gene_flags['cancerGeneListMatch'] = convert_to_rid_set(cancer_gene_flags['cancerGene'])
|
|
522
640
|
|
|
523
641
|
logger.info('fetching therapeutic associated genes lists')
|
|
524
642
|
gene_flags['therapeuticAssociated'] = convert_to_rid_set(
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Functions which return Variants from GraphKB which match some input variant definition
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from typing import Dict, List, Optional,
|
|
5
|
+
from typing import Dict, List, Optional, Union, cast
|
|
6
6
|
|
|
7
7
|
from pori_python.types import (
|
|
8
8
|
BasicPosition,
|
|
@@ -31,14 +31,7 @@ from .util import (
|
|
|
31
31
|
looks_like_rid,
|
|
32
32
|
stringifyVariant,
|
|
33
33
|
)
|
|
34
|
-
from .vocab import
|
|
35
|
-
get_equivalent_terms,
|
|
36
|
-
get_term_by_name,
|
|
37
|
-
get_term_tree,
|
|
38
|
-
get_terms_set,
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
FEATURES_CACHE: Set[str] = set()
|
|
34
|
+
from .vocab import get_equivalent_terms, get_term_by_name, get_term_tree, get_terms_set
|
|
42
35
|
|
|
43
36
|
|
|
44
37
|
def get_equivalent_features(
|
|
@@ -97,8 +90,6 @@ def get_equivalent_features(
|
|
|
97
90
|
filters.append(
|
|
98
91
|
{'OR': [{'sourceIdVersion': source_id_version}, {'sourceIdVersion': None}]}
|
|
99
92
|
)
|
|
100
|
-
elif FEATURES_CACHE and gene_name.lower() not in FEATURES_CACHE and not ignore_cache:
|
|
101
|
-
return []
|
|
102
93
|
else:
|
|
103
94
|
filters.append({'OR': [{'sourceId': gene_name}, {'name': gene_name}]})
|
|
104
95
|
|
|
@@ -111,22 +102,6 @@ def get_equivalent_features(
|
|
|
111
102
|
)
|
|
112
103
|
|
|
113
104
|
|
|
114
|
-
def cache_missing_features(conn: GraphKBConnection) -> None:
|
|
115
|
-
"""
|
|
116
|
-
Create a cache of features that exist to avoid repeatedly querying
|
|
117
|
-
for missing features
|
|
118
|
-
"""
|
|
119
|
-
genes = cast(
|
|
120
|
-
List[Ontology],
|
|
121
|
-
conn.query({'target': 'Feature', 'returnProperties': ['name', 'sourceId'], 'neighbors': 0}),
|
|
122
|
-
)
|
|
123
|
-
for gene in genes:
|
|
124
|
-
if gene['name']:
|
|
125
|
-
FEATURES_CACHE.add(gene['name'].lower())
|
|
126
|
-
if gene['sourceId']:
|
|
127
|
-
FEATURES_CACHE.add(gene['sourceId'].lower())
|
|
128
|
-
|
|
129
|
-
|
|
130
105
|
def match_category_variant(
|
|
131
106
|
conn: GraphKBConnection,
|
|
132
107
|
reference_name: str,
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
from requests.adapters import HTTPAdapter
|
|
3
|
-
|
|
4
|
-
import hashlib
|
|
5
1
|
import json
|
|
6
2
|
import logging
|
|
7
3
|
import os
|
|
8
4
|
import re
|
|
9
5
|
import time
|
|
10
6
|
from datetime import datetime
|
|
11
|
-
from typing import
|
|
12
|
-
from urllib3.util.retry import Retry
|
|
7
|
+
from typing import Dict, Iterable, List, Optional, Union, cast
|
|
13
8
|
from urllib.parse import urlsplit
|
|
14
9
|
|
|
10
|
+
import requests
|
|
11
|
+
from requests.adapters import HTTPAdapter
|
|
12
|
+
from requests_cache import CachedSession
|
|
13
|
+
from requests_ratelimiter import LimiterAdapter
|
|
14
|
+
from urllib3.util.retry import Retry
|
|
15
|
+
|
|
15
16
|
from pori_python.types import ParsedVariant, PositionalVariant, Record
|
|
16
17
|
|
|
17
18
|
from .constants import DEFAULT_LIMIT, TYPES_TO_NOTATION, AA_3to1_MAPPING
|
|
18
19
|
|
|
19
|
-
QUERY_CACHE: Dict[Any, Any] = {}
|
|
20
|
-
|
|
21
20
|
# name the logger after the package to make it simple to disable for packages using this one as a dependency
|
|
22
21
|
# https://stackoverflow.com/questions/11029717/how-do-i-disable-log-messages-from-the-requests-library
|
|
23
22
|
|
|
24
23
|
logger = logging.getLogger('graphkb')
|
|
24
|
+
DEFAULT_LIMITER = LimiterAdapter(per_second=3)
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def convert_to_rid_list(records: Iterable[Record]) -> List[str]:
|
|
@@ -88,11 +88,17 @@ def millis_interval(start: datetime, end: datetime) -> int:
|
|
|
88
88
|
return millis
|
|
89
89
|
|
|
90
90
|
|
|
91
|
-
def
|
|
92
|
-
"""
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
def cache_filter(response):
|
|
92
|
+
"""
|
|
93
|
+
Only cache POST requests when they are for the /query endpoint. Otherwise they are creating content
|
|
94
|
+
and should not be cached
|
|
95
|
+
"""
|
|
96
|
+
request = response.request
|
|
97
|
+
|
|
98
|
+
if request.method == 'POST':
|
|
99
|
+
return request.url.rstrip('/').endswith('/query')
|
|
100
|
+
|
|
101
|
+
return True
|
|
96
102
|
|
|
97
103
|
|
|
98
104
|
class GraphKBConnection:
|
|
@@ -102,23 +108,89 @@ class GraphKBConnection:
|
|
|
102
108
|
username: str = '',
|
|
103
109
|
password: str = '',
|
|
104
110
|
use_global_cache: bool = True,
|
|
111
|
+
cache_name: str = '',
|
|
112
|
+
only_if_cached: bool = False,
|
|
113
|
+
session: Optional[requests.Session] = None,
|
|
114
|
+
limiter_kwargs: Optional[dict] = None,
|
|
115
|
+
**session_kwargs,
|
|
105
116
|
):
|
|
106
|
-
|
|
107
|
-
|
|
117
|
+
"""
|
|
118
|
+
Docstring for __init__
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
- use_global_cache: cache requests across all requests to GKB
|
|
122
|
+
- cache_name: Path or connection URL to the database which stors the requests cache. see https://requests-cache.readthedocs.io/en/v0.6.4/user_guide.html#cache-name
|
|
123
|
+
- only_if_cached: this will set the cache-control header for all requests to only-if-cached which will raise 504 errors if a request does not exist in the cache already rather than making a new network request
|
|
124
|
+
"""
|
|
125
|
+
session_cls = requests.Session
|
|
126
|
+
if session is not None:
|
|
127
|
+
if limiter_kwargs is not None:
|
|
128
|
+
raise NotImplementedError('cannot add limiter to an existing session')
|
|
129
|
+
if use_global_cache:
|
|
130
|
+
raise NotImplementedError(
|
|
131
|
+
'the use_global_cache parameter should not be used with a custom session'
|
|
132
|
+
)
|
|
133
|
+
if cache_name:
|
|
134
|
+
raise NotImplementedError(
|
|
135
|
+
'cache_name should not be used with a custom input session'
|
|
136
|
+
)
|
|
137
|
+
if not use_global_cache and cache_name:
|
|
138
|
+
raise NotImplementedError('cache_name only applies when use_global_cache is True')
|
|
139
|
+
if only_if_cached and not use_global_cache:
|
|
140
|
+
raise NotImplementedError(
|
|
141
|
+
'only_if_cached=True and use_global_cache=False are conflicting settings'
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
if use_global_cache:
|
|
145
|
+
session_cls = CachedSession
|
|
146
|
+
if not cache_name:
|
|
147
|
+
session_kwargs['backend'] = 'memory'
|
|
148
|
+
else:
|
|
149
|
+
session_kwargs['cache_name'] = cache_name
|
|
150
|
+
session_kwargs['allowable_methods'] = ['GET', 'POST']
|
|
151
|
+
session_kwargs['filter_fn'] = cache_filter
|
|
152
|
+
session_kwargs['cache_control'] = True
|
|
153
|
+
|
|
154
|
+
if 'PYTEST_CURRENT_TEST' in os.environ or only_if_cached:
|
|
155
|
+
if limiter_kwargs is not None:
|
|
156
|
+
logging.warning(
|
|
157
|
+
'rate limiting is by default turned off for tests and cache-only queries. Setting limiter to None'
|
|
158
|
+
)
|
|
159
|
+
limiter_kwargs = None
|
|
160
|
+
|
|
161
|
+
if not session:
|
|
162
|
+
self.http = session_cls(**session_kwargs)
|
|
163
|
+
else:
|
|
164
|
+
self.http = session
|
|
165
|
+
|
|
166
|
+
max_retries = Retry(
|
|
108
167
|
total=100,
|
|
109
168
|
connect=5,
|
|
110
169
|
status=5,
|
|
111
170
|
backoff_factor=5,
|
|
112
171
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
113
172
|
)
|
|
114
|
-
|
|
173
|
+
|
|
174
|
+
if limiter_kwargs:
|
|
175
|
+
adapter = LimiterAdapter(max_retries=max_retries, **limiter_kwargs)
|
|
176
|
+
else:
|
|
177
|
+
adapter = HTTPAdapter(max_retries=max_retries)
|
|
178
|
+
|
|
179
|
+
if not only_if_cached:
|
|
180
|
+
# do not need adapters at all for cache-only hits
|
|
181
|
+
self.http.mount('http://', adapter)
|
|
182
|
+
self.http.mount('https://', adapter)
|
|
183
|
+
self.only_if_cached = only_if_cached
|
|
184
|
+
|
|
115
185
|
self.token = ''
|
|
116
186
|
self.token_kc = ''
|
|
117
187
|
self.url = url
|
|
118
188
|
self.username = username
|
|
119
189
|
self.password = password
|
|
120
|
-
self.headers = {
|
|
121
|
-
|
|
190
|
+
self.headers = {
|
|
191
|
+
'Accept': 'application/json',
|
|
192
|
+
'Content-Type': 'application/json',
|
|
193
|
+
}
|
|
122
194
|
self.request_count = 0
|
|
123
195
|
self.first_request: Optional[datetime] = None
|
|
124
196
|
self.last_request: Optional[datetime] = None
|
|
@@ -137,7 +209,16 @@ class GraphKBConnection:
|
|
|
137
209
|
return self.request_count * 1000 / msec
|
|
138
210
|
return None
|
|
139
211
|
|
|
140
|
-
def request(
|
|
212
|
+
def request(
|
|
213
|
+
self,
|
|
214
|
+
endpoint: str,
|
|
215
|
+
method: str = 'GET',
|
|
216
|
+
headers: Optional[dict[str, str]] = None,
|
|
217
|
+
ignore_cache=False,
|
|
218
|
+
force_refresh=False,
|
|
219
|
+
only_if_cached=False,
|
|
220
|
+
**kwargs,
|
|
221
|
+
) -> Dict:
|
|
141
222
|
"""Request wrapper to handle adding common headers and logging.
|
|
142
223
|
|
|
143
224
|
Args:
|
|
@@ -147,6 +228,13 @@ class GraphKBConnection:
|
|
|
147
228
|
Returns:
|
|
148
229
|
dict: the json response as a python dict
|
|
149
230
|
"""
|
|
231
|
+
if headers is None:
|
|
232
|
+
headers = {}
|
|
233
|
+
|
|
234
|
+
if ignore_cache or force_refresh:
|
|
235
|
+
headers['Cache-Control'] = 'no-cache'
|
|
236
|
+
elif only_if_cached or self.only_if_cached:
|
|
237
|
+
headers['Cache-Control'] = 'only-if-cached'
|
|
150
238
|
url = join_url(self.url, endpoint)
|
|
151
239
|
self.request_count += 1
|
|
152
240
|
connect_timeout = 7
|
|
@@ -158,6 +246,11 @@ class GraphKBConnection:
|
|
|
158
246
|
if endpoint in ['query', 'parse']:
|
|
159
247
|
timeout = (connect_timeout, read_timeout)
|
|
160
248
|
|
|
249
|
+
request_headers = {}
|
|
250
|
+
request_headers.update(self.headers)
|
|
251
|
+
if headers is not None:
|
|
252
|
+
request_headers.update(headers)
|
|
253
|
+
|
|
161
254
|
start_time = datetime.now()
|
|
162
255
|
|
|
163
256
|
if not self.first_request:
|
|
@@ -176,11 +269,12 @@ class GraphKBConnection:
|
|
|
176
269
|
try:
|
|
177
270
|
if need_refresh_login:
|
|
178
271
|
self.refresh_login()
|
|
272
|
+
request_headers.update(self.headers)
|
|
179
273
|
need_refresh_login = False
|
|
180
274
|
|
|
181
275
|
self.request_count += 1
|
|
182
|
-
resp =
|
|
183
|
-
method, url, headers=
|
|
276
|
+
resp = self.http.request(
|
|
277
|
+
method, url, headers=request_headers, timeout=timeout, **kwargs
|
|
184
278
|
)
|
|
185
279
|
if resp.status_code == 401 or resp.status_code == 403:
|
|
186
280
|
logger.debug(f'/{endpoint} - {resp.status_code} - retrying')
|
|
@@ -293,39 +387,29 @@ class GraphKBConnection:
|
|
|
293
387
|
def refresh_login(self) -> None:
|
|
294
388
|
self.login(self.username, self.password)
|
|
295
389
|
|
|
296
|
-
def set_cache_data(self, request_body: Dict, result: List[Record]) -> None:
|
|
297
|
-
"""Explicitly add a query to the cache."""
|
|
298
|
-
hash_code = cache_key(request_body)
|
|
299
|
-
self.cache[hash_code] = result
|
|
300
|
-
|
|
301
390
|
def query(
|
|
302
391
|
self,
|
|
303
392
|
request_body: Dict = {},
|
|
304
393
|
paginate: bool = True,
|
|
305
|
-
ignore_cache: bool = False,
|
|
306
|
-
force_refresh: bool = False,
|
|
307
394
|
limit: int = DEFAULT_LIMIT,
|
|
395
|
+
**kwargs,
|
|
308
396
|
) -> List[Record]:
|
|
309
397
|
"""
|
|
310
398
|
Query GraphKB
|
|
311
399
|
"""
|
|
312
|
-
result: List[Record] = []
|
|
313
|
-
hash_code = ''
|
|
314
|
-
|
|
315
|
-
if not ignore_cache and paginate:
|
|
316
|
-
hash_code = cache_key(request_body)
|
|
317
|
-
if hash_code in self.cache and not force_refresh:
|
|
318
|
-
return self.cache[hash_code]
|
|
319
400
|
|
|
401
|
+
result: List[Record] = []
|
|
320
402
|
while True:
|
|
321
|
-
content = self.post(
|
|
403
|
+
content = self.post(
|
|
404
|
+
'query',
|
|
405
|
+
data={**request_body, 'limit': limit, 'skip': len(result)},
|
|
406
|
+
**kwargs,
|
|
407
|
+
)
|
|
322
408
|
records = content['result']
|
|
323
409
|
result.extend(records)
|
|
324
410
|
if len(records) < limit or not paginate:
|
|
325
411
|
break
|
|
326
412
|
|
|
327
|
-
if not ignore_cache and paginate:
|
|
328
|
-
self.cache[hash_code] = result
|
|
329
413
|
return result
|
|
330
414
|
|
|
331
415
|
def parse(self, hgvs_string: str, requireFeatures: bool = False) -> ParsedVariant:
|
|
@@ -354,6 +438,74 @@ class GraphKBConnection:
|
|
|
354
438
|
raise AssertionError(f'Unable to unqiuely identify source with name {name}')
|
|
355
439
|
return source[0]
|
|
356
440
|
|
|
441
|
+
@property
|
|
442
|
+
def version(self) -> Dict[str, str]:
|
|
443
|
+
"""
|
|
444
|
+
Retrieve GraphKB components version
|
|
445
|
+
|
|
446
|
+
Returns:
|
|
447
|
+
Dict[str, str]: component keys with version values
|
|
448
|
+
|
|
449
|
+
e.g. > {"api":"3.17.3","db":"production","parser":"2.1.0","schema":"4.1.1"}
|
|
450
|
+
"""
|
|
451
|
+
return self.request('version')
|
|
452
|
+
|
|
453
|
+
def get_related_records(
|
|
454
|
+
self,
|
|
455
|
+
base: Union[str, List[str]],
|
|
456
|
+
ontology: str,
|
|
457
|
+
subgraphType: str,
|
|
458
|
+
returnProperties: Optional[List[str]] = None,
|
|
459
|
+
) -> List[Record]:
|
|
460
|
+
"""
|
|
461
|
+
Given some base node RIDs, an ontology class and a subgraph type,
|
|
462
|
+
leverage the subgraphs route to return the list of related nodes.
|
|
463
|
+
|
|
464
|
+
Args:
|
|
465
|
+
base: the base node RIDs to start the graph traversal from
|
|
466
|
+
ontology: the ontology class to traverse
|
|
467
|
+
subgraphType: the type of traversal. See options in API specs
|
|
468
|
+
returnProperties: additional record properties to return
|
|
469
|
+
|
|
470
|
+
Returns:
|
|
471
|
+
list of related node record(s) traversed
|
|
472
|
+
"""
|
|
473
|
+
related = self.post(
|
|
474
|
+
uri=f'/subgraphs/{ontology}',
|
|
475
|
+
data={
|
|
476
|
+
'base': base if isinstance(base, list) else [base],
|
|
477
|
+
'subgraphType': subgraphType,
|
|
478
|
+
'returnProperties': returnProperties or [],
|
|
479
|
+
},
|
|
480
|
+
)
|
|
481
|
+
return related['result']['g']['nodes']
|
|
482
|
+
|
|
483
|
+
def get_related_terms(
|
|
484
|
+
self,
|
|
485
|
+
terms: Union[str, List[str]],
|
|
486
|
+
ontology: str = 'Vocabulary',
|
|
487
|
+
subgraphType: str = 'similar',
|
|
488
|
+
) -> List[str]:
|
|
489
|
+
"""
|
|
490
|
+
Given some base term name(s), an ontology class and a subgraph type,
|
|
491
|
+
leverage the subgraphs route to return the list of related term name(s)
|
|
492
|
+
|
|
493
|
+
Args:
|
|
494
|
+
terms: the base term name(s) to start the graph traversal from
|
|
495
|
+
ontology: the ontology class to traverse
|
|
496
|
+
subgraphType: the type of traversal
|
|
497
|
+
|
|
498
|
+
Returns:
|
|
499
|
+
list of related term name(s)
|
|
500
|
+
"""
|
|
501
|
+
rids = convert_to_rid_list(self.query({'target': ontology, 'filters': {'name': terms}}))
|
|
502
|
+
nodes = self.get_related_records(
|
|
503
|
+
base=rids,
|
|
504
|
+
ontology=ontology,
|
|
505
|
+
subgraphType=subgraphType,
|
|
506
|
+
)
|
|
507
|
+
return [x['name'] for x in nodes.values()]
|
|
508
|
+
|
|
357
509
|
|
|
358
510
|
def get_rid(conn: GraphKBConnection, target: str, name: str) -> str:
|
|
359
511
|
"""
|
|
@@ -432,7 +584,9 @@ def stripDisplayName(displayName: str, withRef: bool = True, withRefSeq: bool =
|
|
|
432
584
|
|
|
433
585
|
|
|
434
586
|
def stringifyVariant(
|
|
435
|
-
variant: Union[PositionalVariant, ParsedVariant],
|
|
587
|
+
variant: Union[PositionalVariant, ParsedVariant],
|
|
588
|
+
withRef: bool = True,
|
|
589
|
+
withRefSeq: bool = True,
|
|
436
590
|
) -> str:
|
|
437
591
|
"""
|
|
438
592
|
Convert variant record to a string representation (displayName/hgvs)
|