pyxetabase 3.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyxetabase might be problematic. Click here for more details.
- pyopencga/__init__.py +0 -0
- pyopencga/commons.py +347 -0
- pyopencga/exceptions.py +8 -0
- pyopencga/opencga_client.py +334 -0
- pyopencga/opencga_config.py +160 -0
- pyopencga/rest_clients/__init__.py +0 -0
- pyopencga/rest_clients/_parent_rest_clients.py +110 -0
- pyopencga/rest_clients/admin_client.py +173 -0
- pyopencga/rest_clients/alignment_client.py +373 -0
- pyopencga/rest_clients/clinical_analysis_client.py +1279 -0
- pyopencga/rest_clients/cohort_client.py +338 -0
- pyopencga/rest_clients/disease_panel_client.py +352 -0
- pyopencga/rest_clients/family_client.py +355 -0
- pyopencga/rest_clients/file_client.py +698 -0
- pyopencga/rest_clients/ga4gh_client.py +86 -0
- pyopencga/rest_clients/individual_client.py +435 -0
- pyopencga/rest_clients/job_client.py +415 -0
- pyopencga/rest_clients/meta_client.py +83 -0
- pyopencga/rest_clients/organization_client.py +206 -0
- pyopencga/rest_clients/project_client.py +128 -0
- pyopencga/rest_clients/sample_client.py +446 -0
- pyopencga/rest_clients/study_client.py +433 -0
- pyopencga/rest_clients/user_client.py +192 -0
- pyopencga/rest_clients/variant_client.py +1378 -0
- pyopencga/rest_clients/variant_operation_client.py +719 -0
- pyopencga/rest_clients/workflow_client.py +263 -0
- pyopencga/rest_response.py +220 -0
- pyopencga/retry.py +57 -0
- pyxetabase/__init__.py +0 -0
- pyxetabase/commons.py +347 -0
- pyxetabase/exceptions.py +8 -0
- pyxetabase/opencga_client.py +344 -0
- pyxetabase/opencga_config.py +160 -0
- pyxetabase/rest_clients/__init__.py +0 -0
- pyxetabase/rest_clients/_parent_rest_clients.py +110 -0
- pyxetabase/rest_clients/admin_client.py +173 -0
- pyxetabase/rest_clients/alignment_client.py +373 -0
- pyxetabase/rest_clients/clinical_analysis_client.py +1273 -0
- pyxetabase/rest_clients/cohort_client.py +338 -0
- pyxetabase/rest_clients/cvdb_client.py +2258 -0
- pyxetabase/rest_clients/disease_panel_client.py +352 -0
- pyxetabase/rest_clients/family_client.py +355 -0
- pyxetabase/rest_clients/federation_client.py +133 -0
- pyxetabase/rest_clients/file_client.py +698 -0
- pyxetabase/rest_clients/ga4gh_client.py +86 -0
- pyxetabase/rest_clients/individual_client.py +435 -0
- pyxetabase/rest_clients/job_client.py +415 -0
- pyxetabase/rest_clients/meta_client.py +83 -0
- pyxetabase/rest_clients/organization_client.py +206 -0
- pyxetabase/rest_clients/project_client.py +128 -0
- pyxetabase/rest_clients/sample_client.py +446 -0
- pyxetabase/rest_clients/study_client.py +433 -0
- pyxetabase/rest_clients/user_client.py +212 -0
- pyxetabase/rest_clients/variant_client.py +1378 -0
- pyxetabase/rest_clients/variant_operation_client.py +719 -0
- pyxetabase/rest_clients/workflow_client.py +263 -0
- pyxetabase/rest_response.py +220 -0
- pyxetabase/retry.py +57 -0
- pyxetabase-3.1.0.dev0.dist-info/METADATA +159 -0
- pyxetabase-3.1.0.dev0.dist-info/RECORD +63 -0
- pyxetabase-3.1.0.dev0.dist-info/WHEEL +5 -0
- pyxetabase-3.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- pyxetabase-3.1.0.dev0.dist-info/top_level.txt +1 -0
pyopencga/__init__.py
ADDED
|
File without changes
|
pyopencga/commons.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import threading
|
|
3
|
+
from time import sleep
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
import requests
|
|
7
|
+
|
|
8
|
+
from pyopencga.exceptions import OpencgaInvalidToken, OpencgaAuthorisationError
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from Queue import Queue
|
|
12
|
+
except ImportError:
|
|
13
|
+
from queue import Queue
|
|
14
|
+
|
|
15
|
+
_CALL_BATCH_SIZE = 2000
|
|
16
|
+
_NUM_THREADS_DEFAULT = 4
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def deprecated(func):
|
|
20
|
+
"""Prints a warning for functions marked as deprecated"""
|
|
21
|
+
def new_func(*args, **kwargs):
|
|
22
|
+
warnings.simplefilter('always', DeprecationWarning) # turn off filter
|
|
23
|
+
warnings.warn('Call to deprecated function "{}".'.format(func.__name__),
|
|
24
|
+
category=DeprecationWarning, stacklevel=2)
|
|
25
|
+
warnings.simplefilter('default', DeprecationWarning) # reset filter
|
|
26
|
+
return func(*args, **kwargs)
|
|
27
|
+
return new_func
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def snake_to_camel_case(text):
|
|
31
|
+
"""Converts snake_case to camelCase"""
|
|
32
|
+
components = text.split('_')
|
|
33
|
+
return components[0] + ''.join(x.title() for x in components[1:])
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _create_rest_url(host, version, sid, category, resource, subcategory=None, query_id=None,
|
|
37
|
+
second_query_id=None, options=None):
|
|
38
|
+
"""Creates the URL for querying the REST service"""
|
|
39
|
+
|
|
40
|
+
# Creating the basic URL
|
|
41
|
+
url = ('/'.join([host,
|
|
42
|
+
'webservices/rest',
|
|
43
|
+
version,
|
|
44
|
+
category
|
|
45
|
+
]))
|
|
46
|
+
|
|
47
|
+
# If subcategory is queried, query_id can be absent
|
|
48
|
+
if query_id is not None:
|
|
49
|
+
url += '/' + query_id
|
|
50
|
+
|
|
51
|
+
if subcategory is not None:
|
|
52
|
+
url += '/' + subcategory
|
|
53
|
+
|
|
54
|
+
if second_query_id is not None:
|
|
55
|
+
url += '/' + second_query_id
|
|
56
|
+
|
|
57
|
+
url += '/' + resource
|
|
58
|
+
|
|
59
|
+
header = {"Accept-Encoding": "gzip"}
|
|
60
|
+
if sid is not None:
|
|
61
|
+
header['Authorization'] = 'Bearer {}'.format(sid)
|
|
62
|
+
|
|
63
|
+
# Checking optional params
|
|
64
|
+
if options is not None:
|
|
65
|
+
opts = []
|
|
66
|
+
for k, v in options.items():
|
|
67
|
+
k = snake_to_camel_case(k)
|
|
68
|
+
if k == 'debug':
|
|
69
|
+
continue
|
|
70
|
+
if isinstance(v, list):
|
|
71
|
+
opts.append(k + '=' + ','.join(map(str, v)))
|
|
72
|
+
else:
|
|
73
|
+
opts.append(k + '=' + str(v))
|
|
74
|
+
if opts:
|
|
75
|
+
url += '?' + '&'.join(opts)
|
|
76
|
+
return url, header
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _fetch(config, sid, category, resource, method, subcategory=None, query_id=None,
|
|
80
|
+
second_query_id=None, data=None, options=None):
|
|
81
|
+
"""Queries the REST service retrieving results until exhaustion or limit"""
|
|
82
|
+
# HERE BE DRAGONS
|
|
83
|
+
final_response = None
|
|
84
|
+
|
|
85
|
+
# Setting up skip and limit default parameters
|
|
86
|
+
call_skip = 0
|
|
87
|
+
call_limit = 1000
|
|
88
|
+
max_limit = None
|
|
89
|
+
if options is None:
|
|
90
|
+
opts = {'skip': call_skip, 'limit': call_limit}
|
|
91
|
+
else:
|
|
92
|
+
opts = options.copy() # Do not modify original data!
|
|
93
|
+
if 'skip' not in opts:
|
|
94
|
+
opts['skip'] = call_skip
|
|
95
|
+
# If 'limit' is specified, a maximum of 'limit' results will be returned
|
|
96
|
+
if 'limit' in opts:
|
|
97
|
+
max_limit = int(opts['limit'])
|
|
98
|
+
# Server must be always queried for results in groups of 1000
|
|
99
|
+
opts['limit'] = call_limit
|
|
100
|
+
|
|
101
|
+
# If there is a query_id, the next variables will be used
|
|
102
|
+
total_id_list = [] # All initial ids
|
|
103
|
+
next_id_list = [] # Ids which should be queried again for more results
|
|
104
|
+
next_id_indexes = [] # Ids position in the final response
|
|
105
|
+
if query_id is not None:
|
|
106
|
+
total_id_list = query_id.split(',')
|
|
107
|
+
|
|
108
|
+
# If some query has more than 'call_limit' results, the server will be
|
|
109
|
+
# queried again to retrieve the next 'call_limit results'
|
|
110
|
+
call = True
|
|
111
|
+
current_query_id = None # Current REST query
|
|
112
|
+
current_id_list = None # Current list of ids
|
|
113
|
+
time_out_counter = 0 # Number of times a query is repeated due to time-out
|
|
114
|
+
while call:
|
|
115
|
+
# Check 'limit' parameter if there is a maximum limit of results
|
|
116
|
+
if max_limit is not None and max_limit <= call_limit:
|
|
117
|
+
opts['limit'] = max_limit
|
|
118
|
+
|
|
119
|
+
# Updating query_id and list of ids to query
|
|
120
|
+
if query_id is not None:
|
|
121
|
+
if current_query_id is None:
|
|
122
|
+
current_query_id = query_id
|
|
123
|
+
current_id_list = total_id_list
|
|
124
|
+
current_id_indexes = range(len(total_id_list))
|
|
125
|
+
else:
|
|
126
|
+
current_query_id = ','.join(next_id_list)
|
|
127
|
+
current_id_list = next_id_list
|
|
128
|
+
current_id_indexes = next_id_indexes
|
|
129
|
+
|
|
130
|
+
# Retrieving url
|
|
131
|
+
url, header = _create_rest_url(host=config.host,
|
|
132
|
+
version=config.version,
|
|
133
|
+
category=category,
|
|
134
|
+
sid=sid,
|
|
135
|
+
subcategory=subcategory,
|
|
136
|
+
query_id=current_query_id,
|
|
137
|
+
second_query_id=second_query_id,
|
|
138
|
+
resource=resource,
|
|
139
|
+
options=opts)
|
|
140
|
+
|
|
141
|
+
# DEBUG param
|
|
142
|
+
if opts is not None and 'debug' in opts and opts['debug']:
|
|
143
|
+
sys.stderr.write(url + '\n')
|
|
144
|
+
|
|
145
|
+
# Getting REST response
|
|
146
|
+
if method == 'get':
|
|
147
|
+
try:
|
|
148
|
+
r = requests.get(url, headers=header, cookies=config.cookies)
|
|
149
|
+
except requests.exceptions.ConnectionError:
|
|
150
|
+
sleep(1)
|
|
151
|
+
r = requests.get(url, headers=header, cookies=config.cookies)
|
|
152
|
+
elif method == 'post':
|
|
153
|
+
try:
|
|
154
|
+
r = requests.post(url, json=data, headers=header, cookies=config.cookies)
|
|
155
|
+
except requests.exceptions.ConnectionError:
|
|
156
|
+
sleep(1)
|
|
157
|
+
r = requests.post(url, json=data, headers=header, cookies=config.cookies)
|
|
158
|
+
elif method == 'delete':
|
|
159
|
+
try:
|
|
160
|
+
r = requests.delete(url, headers=header, cookies=config.cookies)
|
|
161
|
+
except requests.exceptions.ConnectionError:
|
|
162
|
+
sleep(1)
|
|
163
|
+
r = requests.delete(url, headers=header, cookies=config.cookies)
|
|
164
|
+
else:
|
|
165
|
+
raise NotImplementedError('method: ' + method + ' not implemented.')
|
|
166
|
+
|
|
167
|
+
if r.status_code == 504: # Gateway Time-out
|
|
168
|
+
if time_out_counter == 99:
|
|
169
|
+
msg = 'Server not responding in time'
|
|
170
|
+
raise requests.ConnectionError(msg)
|
|
171
|
+
time_out_counter += 1
|
|
172
|
+
continue
|
|
173
|
+
time_out_counter = 0
|
|
174
|
+
|
|
175
|
+
if r.status_code == 401:
|
|
176
|
+
raise OpencgaInvalidToken(r.content)
|
|
177
|
+
elif r.status_code == 403:
|
|
178
|
+
raise OpencgaAuthorisationError(r.content)
|
|
179
|
+
elif r.status_code != 200:
|
|
180
|
+
raise Exception(r.content)
|
|
181
|
+
|
|
182
|
+
if r.headers['Content-Type'] == 'application/json':
|
|
183
|
+
try:
|
|
184
|
+
response = r.json()
|
|
185
|
+
|
|
186
|
+
# TODO Remove deprecated response and result in future release. Added for backwards compatibility
|
|
187
|
+
if 'response' in response:
|
|
188
|
+
response['responses'] = response['response']
|
|
189
|
+
for query_result in response['responses']:
|
|
190
|
+
if 'result' in query_result:
|
|
191
|
+
query_result['results'] = query_result['result']
|
|
192
|
+
|
|
193
|
+
except ValueError:
|
|
194
|
+
raise ValueError('Bad JSON format retrieved from server')
|
|
195
|
+
elif r.headers['Content-Type'] == 'application/octet-stream':
|
|
196
|
+
return r.content
|
|
197
|
+
else:
|
|
198
|
+
raise ValueError('Unexpected content type retrieved from server ("{}"): "{}"'.format(
|
|
199
|
+
r.headers['Content-Type'], r.content)
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# Setting up final_response
|
|
203
|
+
if final_response is None:
|
|
204
|
+
final_response = response
|
|
205
|
+
# Concatenating results
|
|
206
|
+
else:
|
|
207
|
+
if query_id is not None:
|
|
208
|
+
for index, res in enumerate(response['responses']):
|
|
209
|
+
id_index = current_id_indexes[index]
|
|
210
|
+
final_response[id_index]['results'] += res['results']
|
|
211
|
+
else:
|
|
212
|
+
final_response['responses'][0]['results'] += response['responses'][0]['results']
|
|
213
|
+
|
|
214
|
+
if query_id is not None:
|
|
215
|
+
# Checking which ids are completely retrieved
|
|
216
|
+
next_id_list = []
|
|
217
|
+
next_id_indexes = []
|
|
218
|
+
for index, res in enumerate(response['responses']):
|
|
219
|
+
if res['numResults'] == call_limit:
|
|
220
|
+
next_id_list.append(current_id_list[index])
|
|
221
|
+
next_id_indexes.append(current_id_indexes[index])
|
|
222
|
+
# Ending REST calling when there are no more ids to retrieve
|
|
223
|
+
if not next_id_list:
|
|
224
|
+
call = False
|
|
225
|
+
else:
|
|
226
|
+
# Ending REST calling when there are no more results to retrieve
|
|
227
|
+
if response['responses'][0]['numResults'] != call_limit:
|
|
228
|
+
call = False
|
|
229
|
+
|
|
230
|
+
# Skipping the first 'limit' results to retrieve the next ones
|
|
231
|
+
opts['skip'] += call_limit
|
|
232
|
+
|
|
233
|
+
# Subtracting the number of returned results from the maximum goal
|
|
234
|
+
if max_limit is not None:
|
|
235
|
+
max_limit -= call_limit
|
|
236
|
+
# When 'limit' is 0 returns all the results. So, break the loop if 0
|
|
237
|
+
if max_limit == 0:
|
|
238
|
+
break
|
|
239
|
+
|
|
240
|
+
return final_response
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _worker(queue, results, config, sid, category, resource, method, subcategory=None,
|
|
244
|
+
second_query_id=None, data=None, options=None):
|
|
245
|
+
|
|
246
|
+
"""Manages the queue system for the threads"""
|
|
247
|
+
while True:
|
|
248
|
+
# Fetching new element from the queue
|
|
249
|
+
index, query_id = queue.get()
|
|
250
|
+
response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
|
|
251
|
+
resource=resource, method=method, data=data, query_id=query_id,
|
|
252
|
+
second_query_id=second_query_id, options=options)
|
|
253
|
+
# Store data in results at correct index
|
|
254
|
+
results[index] = response
|
|
255
|
+
# Signaling to the queue that task has been processed
|
|
256
|
+
queue.task_done()
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def merge_query_responses(query_response_list):
|
|
260
|
+
final_response = query_response_list[0]
|
|
261
|
+
for i, query_response in enumerate(query_response_list):
|
|
262
|
+
if i != 0:
|
|
263
|
+
final_response['events'] += query_response['events']
|
|
264
|
+
final_response['time'] += query_response['time']
|
|
265
|
+
# final_response['responses'] += response['responses']
|
|
266
|
+
|
|
267
|
+
for key in query_response['params']:
|
|
268
|
+
if final_response['params'][key] != query_response['params'][key]:
|
|
269
|
+
final_response['params'][key] += ',' + query_response['params'][key]
|
|
270
|
+
|
|
271
|
+
for j, query_result in enumerate(query_response['responses']):
|
|
272
|
+
if len(final_response['responses'])-1 < j:
|
|
273
|
+
final_response['responses'] += []
|
|
274
|
+
for key in query_result:
|
|
275
|
+
if key not in final_response['responses'][j]:
|
|
276
|
+
final_response['responses'][j][key] = query_result[key]
|
|
277
|
+
else:
|
|
278
|
+
if isinstance(query_result[key], (int, list)):
|
|
279
|
+
final_response['responses'][j][key] += query_result[key]
|
|
280
|
+
return final_response
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def execute(config, sid, category, resource, method, subcategory=None, query_id=None,
|
|
284
|
+
second_query_id=None, data=None, options=None):
|
|
285
|
+
"""Queries the REST service using multiple threads if needed"""
|
|
286
|
+
|
|
287
|
+
# If query_id is an array, convert to comma-separated string
|
|
288
|
+
if query_id is not None:
|
|
289
|
+
if isinstance(query_id, list):
|
|
290
|
+
query_id = ','.join([str(item) for item in query_id])
|
|
291
|
+
else:
|
|
292
|
+
query_id = str(query_id) # convert to string so we can call this method with int ids
|
|
293
|
+
|
|
294
|
+
# Multithread if the number of queries is greater than _CALL_BATCH_SIZE
|
|
295
|
+
if query_id is None or len(query_id.split(',')) <= _CALL_BATCH_SIZE:
|
|
296
|
+
response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
|
|
297
|
+
resource=resource, method=method, data=data, query_id=query_id,
|
|
298
|
+
second_query_id=second_query_id, options=options)
|
|
299
|
+
return response
|
|
300
|
+
else:
|
|
301
|
+
if options is not None and 'num_threads' in options:
|
|
302
|
+
num_threads = options['num_threads']
|
|
303
|
+
else:
|
|
304
|
+
num_threads = _NUM_THREADS_DEFAULT
|
|
305
|
+
|
|
306
|
+
# Splitting query_id into batches depending on the call batch size
|
|
307
|
+
id_list = query_id.split(',')
|
|
308
|
+
id_batches = [','.join(id_list[x:x + _CALL_BATCH_SIZE])
|
|
309
|
+
for x in range(0, len(id_list), _CALL_BATCH_SIZE)]
|
|
310
|
+
|
|
311
|
+
# Setting up the queue to hold all the id batches
|
|
312
|
+
q = Queue(maxsize=0)
|
|
313
|
+
# Creating a size defined list to store thread results
|
|
314
|
+
res = [''] * len(id_batches)
|
|
315
|
+
|
|
316
|
+
# Setting up the threads
|
|
317
|
+
for thread in range(num_threads):
|
|
318
|
+
t = threading.Thread(target=_worker,
|
|
319
|
+
kwargs={'queue': q,
|
|
320
|
+
'results': res,
|
|
321
|
+
'config': config,
|
|
322
|
+
'sid': sid,
|
|
323
|
+
'category': category,
|
|
324
|
+
'subcategory': subcategory,
|
|
325
|
+
'second_query_id': second_query_id,
|
|
326
|
+
'resource': resource,
|
|
327
|
+
'method': method,
|
|
328
|
+
'data': data,
|
|
329
|
+
'options': options})
|
|
330
|
+
# Setting threads as "daemon" allows main program to exit eventually
|
|
331
|
+
# even if these do not finish correctly
|
|
332
|
+
t.daemon = True
|
|
333
|
+
t.start()
|
|
334
|
+
|
|
335
|
+
# Loading up the queue with index and id batches for each job
|
|
336
|
+
for index, batch in enumerate(id_batches):
|
|
337
|
+
q.put((index, batch)) # Notice this is a tuple
|
|
338
|
+
|
|
339
|
+
# Waiting until the queue has been processed
|
|
340
|
+
q.join()
|
|
341
|
+
|
|
342
|
+
# Joining all the responses into a one final response
|
|
343
|
+
final_query_response = merge_query_responses(res)
|
|
344
|
+
|
|
345
|
+
return final_query_response
|
|
346
|
+
|
|
347
|
+
|
pyopencga/exceptions.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
import getpass
|
|
2
|
+
import time
|
|
3
|
+
import sys
|
|
4
|
+
import re
|
|
5
|
+
if sys.version_info >= (3, 8):
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
else:
|
|
8
|
+
from importlib_metadata import version
|
|
9
|
+
|
|
10
|
+
from pyopencga.opencga_config import ClientConfiguration
|
|
11
|
+
from pyopencga.rest_clients.admin_client import Admin
|
|
12
|
+
from pyopencga.rest_clients.alignment_client import Alignment
|
|
13
|
+
from pyopencga.rest_clients.clinical_analysis_client import ClinicalAnalysis
|
|
14
|
+
from pyopencga.rest_clients.cohort_client import Cohort
|
|
15
|
+
from pyopencga.rest_clients.family_client import Family
|
|
16
|
+
from pyopencga.rest_clients.file_client import File
|
|
17
|
+
from pyopencga.rest_clients.ga4gh_client import GA4GH
|
|
18
|
+
from pyopencga.rest_clients.individual_client import Individual
|
|
19
|
+
from pyopencga.rest_clients.job_client import Job
|
|
20
|
+
from pyopencga.rest_clients.meta_client import Meta
|
|
21
|
+
from pyopencga.rest_clients.disease_panel_client import DiseasePanel
|
|
22
|
+
from pyopencga.rest_clients.project_client import Project
|
|
23
|
+
from pyopencga.rest_clients.sample_client import Sample
|
|
24
|
+
from pyopencga.rest_clients.study_client import Study
|
|
25
|
+
from pyopencga.rest_clients.variant_operation_client import VariantOperation
|
|
26
|
+
from pyopencga.rest_clients.user_client import User
|
|
27
|
+
from pyopencga.rest_clients.variant_client import Variant
|
|
28
|
+
from pyopencga.rest_clients.organization_client import Organization
|
|
29
|
+
from pyopencga.rest_clients.workflow_client import Workflow
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class OpencgaClient(object):
|
|
33
|
+
def __init__(self, configuration, token=None, refresh_token=None, on_retry=None, auto_refresh=True):
|
|
34
|
+
"""
|
|
35
|
+
:param on_retry: callback to be called with client retries an operation.
|
|
36
|
+
It must accept parameters: client, exc_type, exc_val, exc_tb, call
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
if not isinstance(configuration, ClientConfiguration):
|
|
40
|
+
raise ValueError('Expected ClientConfiguration instance')
|
|
41
|
+
|
|
42
|
+
self.configuration = configuration
|
|
43
|
+
self.auto_refresh = auto_refresh
|
|
44
|
+
self.on_retry = on_retry
|
|
45
|
+
self.clients = []
|
|
46
|
+
self._login_handler = None
|
|
47
|
+
self.token = token
|
|
48
|
+
self.refresh_token = refresh_token
|
|
49
|
+
self._create_clients()
|
|
50
|
+
self._check_versions()
|
|
51
|
+
if self.configuration.sso_login:
|
|
52
|
+
self._sso_login()
|
|
53
|
+
|
|
54
|
+
def __enter__(self):
|
|
55
|
+
return self
|
|
56
|
+
|
|
57
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
58
|
+
self.logout()
|
|
59
|
+
|
|
60
|
+
def _check_versions(self):
|
|
61
|
+
# Getting client and server versions
|
|
62
|
+
client_version = version("pyopencga")
|
|
63
|
+
server_version = self.meta.about().get_result(0)['Version'].split('-')[0]
|
|
64
|
+
|
|
65
|
+
ansi_reset = "\033[0m"
|
|
66
|
+
ansi_red = "\033[31m"
|
|
67
|
+
ansi_yellow = "\033[33m"
|
|
68
|
+
if tuple(server_version.split('.')[:2]) < tuple(client_version.split('.')[:2]):
|
|
69
|
+
msg = '[WARNING]: Client version ({}) is higher than server version ({}).' \
|
|
70
|
+
' Some client features may not be implemented in the server.\n'.format(client_version, server_version)
|
|
71
|
+
sys.stdout.write('{}{}{}'.format(ansi_red, msg, ansi_reset))
|
|
72
|
+
elif tuple(server_version.split('.')[:2]) > tuple(client_version.split('.')[:2]):
|
|
73
|
+
msg = '[INFO]: Client version ({}) is lower than server version ({}).' \
|
|
74
|
+
' Some client features may not work as intended.\n'.format(client_version, server_version)
|
|
75
|
+
sys.stdout.write('{}{}{}'.format(ansi_yellow, msg, ansi_reset))
|
|
76
|
+
|
|
77
|
+
def _create_clients(self):
|
|
78
|
+
self.organizations = Organization(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
79
|
+
self.users = User(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
80
|
+
self.projects = Project(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
81
|
+
self.studies = Study(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
82
|
+
self.files = File(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
83
|
+
self.jobs = Job(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
84
|
+
self.samples = Sample(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
85
|
+
self.individuals = Individual(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
86
|
+
self.families = Family(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
87
|
+
self.cohorts = Cohort(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
88
|
+
self.disease_panels = DiseasePanel(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
89
|
+
self.alignments = Alignment(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
90
|
+
self.variants = Variant(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
91
|
+
self.clinical = ClinicalAnalysis(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
92
|
+
self.operations = VariantOperation(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
93
|
+
self.variant_operations = self.operations # DEPRECATED: use 'self.operations'
|
|
94
|
+
self.meta = Meta(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
95
|
+
self.ga4gh = GA4GH(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
96
|
+
self.admin = Admin(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
97
|
+
self.workflows = Workflow(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
|
|
98
|
+
|
|
99
|
+
self.clients = [
|
|
100
|
+
self.organizations, self.users, self.projects, self.studies, self.files, self.jobs,
|
|
101
|
+
self.samples, self.individuals, self.families, self.cohorts,
|
|
102
|
+
self.disease_panels, self.alignments, self.variants, self.clinical, self.operations,
|
|
103
|
+
self.variant_operations, self.meta, self.ga4gh, self.admin, self.workflows
|
|
104
|
+
]
|
|
105
|
+
|
|
106
|
+
for client in self.clients:
|
|
107
|
+
client.on_retry = self.on_retry
|
|
108
|
+
|
|
109
|
+
def _make_login_handler(self, user=None, password=None, organization=None):
|
|
110
|
+
"""
|
|
111
|
+
Returns a closure that performs the log-in. This will be called on retries
|
|
112
|
+
if the current session ever expires.
|
|
113
|
+
The reason for using a closure and not a normal function is that a normal
|
|
114
|
+
function would require storing the password in a field. It is more secure
|
|
115
|
+
not to do so. This way, the password stored in the closure is inaccessible
|
|
116
|
+
to other code
|
|
117
|
+
"""
|
|
118
|
+
def login_handler(refresh=False):
|
|
119
|
+
if refresh:
|
|
120
|
+
data = {'refreshToken': self.refresh_token}
|
|
121
|
+
else:
|
|
122
|
+
data = {'user': user, 'password': password}
|
|
123
|
+
if organization:
|
|
124
|
+
data.update({'organization': organization})
|
|
125
|
+
tokens = User(self.configuration).login(data=data).get_result(0)
|
|
126
|
+
self.token = tokens['token']
|
|
127
|
+
self.refresh_token = tokens['refreshToken']
|
|
128
|
+
return self.token, self.refresh_token
|
|
129
|
+
return login_handler
|
|
130
|
+
|
|
131
|
+
def login(self, user=None, password=None, organization=None):
|
|
132
|
+
if user is not None and password is None:
|
|
133
|
+
password = getpass.getpass()
|
|
134
|
+
if not (user and password):
|
|
135
|
+
raise ValueError('User and password required')
|
|
136
|
+
|
|
137
|
+
self._login_handler = self._make_login_handler(user=user, password=password, organization=organization)
|
|
138
|
+
self._login_handler()
|
|
139
|
+
for client in self.clients:
|
|
140
|
+
client.token = self.token
|
|
141
|
+
client.refresh_token = self.refresh_token
|
|
142
|
+
client.login_handler = self._login_handler
|
|
143
|
+
|
|
144
|
+
def _sso_login(self):
|
|
145
|
+
# Getting token and refresh token from configuration
|
|
146
|
+
self.token = self.configuration.token
|
|
147
|
+
self.refresh_token = self.configuration.token
|
|
148
|
+
|
|
149
|
+
self._login_handler = self._make_login_handler()
|
|
150
|
+
for client in self.clients:
|
|
151
|
+
client.token = self.token
|
|
152
|
+
client.refresh_token = self.refresh_token
|
|
153
|
+
client.login_handler = self._login_handler
|
|
154
|
+
|
|
155
|
+
def logout(self):
|
|
156
|
+
self.token = None
|
|
157
|
+
self.refresh_token = None
|
|
158
|
+
for client in self.clients:
|
|
159
|
+
client.token = self.token
|
|
160
|
+
client.refresh_token = self.refresh_token
|
|
161
|
+
|
|
162
|
+
def wait_for_job(self, response=None, study_id=None, job_id=None, retry_seconds=10):
|
|
163
|
+
if response is not None:
|
|
164
|
+
study_id = response['results'][0]['study']['id']
|
|
165
|
+
job_id = response['results'][0]['id']
|
|
166
|
+
|
|
167
|
+
if response is None and (study_id is None or job_id is None):
|
|
168
|
+
raise ValueError('Argument "response" or arguments "study" and "job_id" must be provided')
|
|
169
|
+
|
|
170
|
+
if len(job_id.split(',')) > 1:
|
|
171
|
+
raise ValueError('Only one job ID is allowed')
|
|
172
|
+
|
|
173
|
+
retry_seconds = retry_seconds if retry_seconds >= 10 else 10
|
|
174
|
+
while True:
|
|
175
|
+
job_info = self.jobs.info(study=study_id, jobs=job_id).get_result(0)
|
|
176
|
+
if job_info['internal']['status']['id'] in ['ERROR', 'ABORTED']:
|
|
177
|
+
raise ValueError('{} ({}): {}'.format(
|
|
178
|
+
job_info['internal']['status']['id'],
|
|
179
|
+
job_info['internal']['status']['date'],
|
|
180
|
+
job_info['internal']['status']['description']
|
|
181
|
+
))
|
|
182
|
+
elif job_info['internal']['status']['id'] in ['DONE']:
|
|
183
|
+
break
|
|
184
|
+
time.sleep(retry_seconds)
|
|
185
|
+
|
|
186
|
+
def _get_help_info(self, client_name=None, parameters=False):
|
|
187
|
+
info = []
|
|
188
|
+
for client in self.clients:
|
|
189
|
+
# Name
|
|
190
|
+
cls_name = type(client).__name__
|
|
191
|
+
client_method = re.sub(r'(?<!^)(?=[A-Z])', '_', cls_name).lower() \
|
|
192
|
+
if cls_name != 'GA4GH' else cls_name.lower()
|
|
193
|
+
client_method = 'get_' + client_method + '_client'
|
|
194
|
+
|
|
195
|
+
if client_name is not None and client_name != cls_name:
|
|
196
|
+
continue
|
|
197
|
+
|
|
198
|
+
# Description and path
|
|
199
|
+
class_docstring = client.__doc__
|
|
200
|
+
cls_desc = re.findall('(This class contains methods .+)\n', class_docstring)[0]
|
|
201
|
+
cls_path = re.findall('PATH: (.+)\n', class_docstring)[0]
|
|
202
|
+
|
|
203
|
+
# Methods
|
|
204
|
+
methods = []
|
|
205
|
+
method_names = [method_name for method_name in dir(client)
|
|
206
|
+
if callable(getattr(client, method_name))
|
|
207
|
+
and not method_name.startswith('_') and method_name != 'login_handler']
|
|
208
|
+
for method_name in method_names:
|
|
209
|
+
if client_name is None:
|
|
210
|
+
continue
|
|
211
|
+
method_docstring = getattr(client, method_name).__doc__
|
|
212
|
+
desc = re.findall('(.+)\n +PATH', method_docstring, re.DOTALL)
|
|
213
|
+
desc = re.sub(' +', ' ', desc[0].replace('\n', ' ').strip())
|
|
214
|
+
path = re.findall('PATH: (.+)\n', method_docstring)[0]
|
|
215
|
+
|
|
216
|
+
args = []
|
|
217
|
+
arguments = re.findall(
|
|
218
|
+
' +:param (.+)', method_docstring, re.DOTALL
|
|
219
|
+
)
|
|
220
|
+
if arguments and parameters:
|
|
221
|
+
arguments = arguments[0].replace('\n', ' ').strip()
|
|
222
|
+
arguments = re.sub(' +', ' ', arguments)
|
|
223
|
+
arguments = arguments.split(' :param ')
|
|
224
|
+
for parameter in arguments:
|
|
225
|
+
param_info = parameter.split(' ', 2)
|
|
226
|
+
args.append({
|
|
227
|
+
'name': param_info[1].rstrip(':'),
|
|
228
|
+
'type': param_info[0],
|
|
229
|
+
'desc': param_info[2]
|
|
230
|
+
})
|
|
231
|
+
methods.append({
|
|
232
|
+
'name': method_name,
|
|
233
|
+
'desc': desc,
|
|
234
|
+
'path': path,
|
|
235
|
+
'params': args
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
info.append(
|
|
239
|
+
{'class_name': cls_name, 'client_method': client_method,
|
|
240
|
+
'desc': cls_desc, 'path': cls_path, 'methods': methods}
|
|
241
|
+
)
|
|
242
|
+
return info
|
|
243
|
+
|
|
244
|
+
def help(self, client_name=None, show_parameters=False):
|
|
245
|
+
help_txt = []
|
|
246
|
+
|
|
247
|
+
info = self._get_help_info(client_name, show_parameters)
|
|
248
|
+
if client_name is None:
|
|
249
|
+
help_txt += ['Available clients:']
|
|
250
|
+
for client in info:
|
|
251
|
+
txt = '{}- {}: {} ({}). USAGE: opencga_client.{}()'
|
|
252
|
+
help_txt += [txt.format(
|
|
253
|
+
' '*4, client['class_name'], client['desc'],
|
|
254
|
+
client['path'], client['client_method']
|
|
255
|
+
)]
|
|
256
|
+
else:
|
|
257
|
+
for client in info:
|
|
258
|
+
help_txt += ['{}: {} ({}). USAGE: opencga_client.{}()'.format(
|
|
259
|
+
client['class_name'], client['desc'], client['path'],
|
|
260
|
+
client['client_method']
|
|
261
|
+
)]
|
|
262
|
+
help_txt += ['{}Available methods:'.format(' '*4)]
|
|
263
|
+
for method in client['methods']:
|
|
264
|
+
help_txt += ['{}- {}: {} ({})'.format(
|
|
265
|
+
' '*8, method['name'], method['desc'], method['path']
|
|
266
|
+
)]
|
|
267
|
+
if not show_parameters:
|
|
268
|
+
continue
|
|
269
|
+
for param in method['params']:
|
|
270
|
+
help_txt += ['{}* {} ({}): {}'.format(
|
|
271
|
+
' ' * 12, param['name'], param['type'],
|
|
272
|
+
param['desc']
|
|
273
|
+
)]
|
|
274
|
+
sys.stdout.write('\n'.join(help_txt) + '\n')
|
|
275
|
+
|
|
276
|
+
def get_organization_client(self):
|
|
277
|
+
return self.organizations
|
|
278
|
+
|
|
279
|
+
def get_user_client(self):
|
|
280
|
+
return self.users
|
|
281
|
+
|
|
282
|
+
def get_project_client(self):
|
|
283
|
+
return self.projects
|
|
284
|
+
|
|
285
|
+
def get_study_client(self):
|
|
286
|
+
return self.studies
|
|
287
|
+
|
|
288
|
+
def get_file_client(self):
|
|
289
|
+
return self.files
|
|
290
|
+
|
|
291
|
+
def get_job_client(self):
|
|
292
|
+
return self.jobs
|
|
293
|
+
|
|
294
|
+
def get_sample_client(self):
|
|
295
|
+
return self.samples
|
|
296
|
+
|
|
297
|
+
def get_individual_client(self):
|
|
298
|
+
return self.individuals
|
|
299
|
+
|
|
300
|
+
def get_family_client(self):
|
|
301
|
+
return self.families
|
|
302
|
+
|
|
303
|
+
def get_cohort_client(self):
|
|
304
|
+
return self.cohorts
|
|
305
|
+
|
|
306
|
+
def get_disease_panel_client(self):
|
|
307
|
+
return self.disease_panels
|
|
308
|
+
|
|
309
|
+
def get_alignment_client(self):
|
|
310
|
+
return self.alignments
|
|
311
|
+
|
|
312
|
+
def get_variant_client(self):
|
|
313
|
+
return self.variants
|
|
314
|
+
|
|
315
|
+
def get_clinical_client(self):
|
|
316
|
+
return self.clinical
|
|
317
|
+
|
|
318
|
+
def get_operations_client(self):
|
|
319
|
+
return self.operations
|
|
320
|
+
|
|
321
|
+
def get_variant_operation_client(self):
|
|
322
|
+
return self.variant_operations
|
|
323
|
+
|
|
324
|
+
def get_meta_client(self):
|
|
325
|
+
return self.meta
|
|
326
|
+
|
|
327
|
+
def get_ga4gh_client(self):
|
|
328
|
+
return self.ga4gh
|
|
329
|
+
|
|
330
|
+
def get_admin_client(self):
|
|
331
|
+
return self.admin
|
|
332
|
+
|
|
333
|
+
def get_workflows_client(self):
|
|
334
|
+
return self.workflows
|