pyxetabase 3.1.0.dev25__py3-none-any.whl → 4.0.0.dev35__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyxetabase/rest_clients/clinical_analysis_client.py +12 -74
- pyxetabase/rest_clients/disease_panel_client.py +0 -13
- pyxetabase/rest_clients/file_client.py +1 -1
- pyxetabase/rest_clients/variant_operation_client.py +1 -2
- {pyxetabase-3.1.0.dev25.dist-info → pyxetabase-4.0.0.dev35.dist-info}/METADATA +1 -1
- pyxetabase-4.0.0.dev35.dist-info/RECORD +35 -0
- pyopencga/__init__.py +0 -0
- pyopencga/commons.py +0 -347
- pyopencga/exceptions.py +0 -8
- pyopencga/opencga_client.py +0 -334
- pyopencga/opencga_config.py +0 -211
- pyopencga/rest_clients/__init__.py +0 -0
- pyopencga/rest_clients/_parent_rest_clients.py +0 -110
- pyopencga/rest_clients/admin_client.py +0 -172
- pyopencga/rest_clients/alignment_client.py +0 -373
- pyopencga/rest_clients/clinical_analysis_client.py +0 -1279
- pyopencga/rest_clients/cohort_client.py +0 -338
- pyopencga/rest_clients/disease_panel_client.py +0 -352
- pyopencga/rest_clients/family_client.py +0 -355
- pyopencga/rest_clients/file_client.py +0 -698
- pyopencga/rest_clients/ga4gh_client.py +0 -86
- pyopencga/rest_clients/individual_client.py +0 -435
- pyopencga/rest_clients/job_client.py +0 -415
- pyopencga/rest_clients/meta_client.py +0 -85
- pyopencga/rest_clients/organization_client.py +0 -216
- pyopencga/rest_clients/project_client.py +0 -128
- pyopencga/rest_clients/sample_client.py +0 -446
- pyopencga/rest_clients/study_client.py +0 -461
- pyopencga/rest_clients/user_client.py +0 -192
- pyopencga/rest_clients/variant_client.py +0 -1378
- pyopencga/rest_clients/variant_operation_client.py +0 -746
- pyopencga/rest_clients/workflow_client.py +0 -263
- pyopencga/rest_response.py +0 -220
- pyopencga/retry.py +0 -57
- pyxetabase-3.1.0.dev25.dist-info/RECORD +0 -63
- {pyxetabase-3.1.0.dev25.dist-info → pyxetabase-4.0.0.dev35.dist-info}/WHEEL +0 -0
- {pyxetabase-3.1.0.dev25.dist-info → pyxetabase-4.0.0.dev35.dist-info}/licenses/LICENSE +0 -0
- {pyxetabase-3.1.0.dev25.dist-info → pyxetabase-4.0.0.dev35.dist-info}/top_level.txt +0 -0
|
@@ -350,32 +350,6 @@ class ClinicalAnalysis(_ParentRestClient):
|
|
|
350
350
|
|
|
351
351
|
return self._get(category='analysis', resource='info', subcategory='clinical/interpretation', second_query_id=interpretations, **options)
|
|
352
352
|
|
|
353
|
-
def run_interpreter_cancer_tiering(self, data=None, **options):
|
|
354
|
-
"""
|
|
355
|
-
Run cancer tiering interpretation analysis.
|
|
356
|
-
PATH: /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run
|
|
357
|
-
|
|
358
|
-
:param dict data: Cancer tiering interpretation analysis params.
|
|
359
|
-
(REQUIRED)
|
|
360
|
-
:param str study: Study [[organization@]project:]study where study and
|
|
361
|
-
project can be either the ID or UUID.
|
|
362
|
-
:param str job_id: Job ID. It must be a unique string within the
|
|
363
|
-
study. An ID will be autogenerated automatically if not provided.
|
|
364
|
-
:param str job_description: Job description.
|
|
365
|
-
:param str job_depends_on: Comma separated list of existing job IDs
|
|
366
|
-
the job will depend on.
|
|
367
|
-
:param str job_tags: Job tags.
|
|
368
|
-
:param str job_scheduled_start_time: Time when the job is scheduled to
|
|
369
|
-
start.
|
|
370
|
-
:param str job_priority: Priority of the job.
|
|
371
|
-
:param bool job_dry_run: Flag indicating that the job will be executed
|
|
372
|
-
in dry-run mode. In this mode, OpenCGA will validate that all
|
|
373
|
-
parameters and prerequisites are correctly set for successful
|
|
374
|
-
execution, but the job will not actually run.
|
|
375
|
-
"""
|
|
376
|
-
|
|
377
|
-
return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/cancerTiering', data=data, **options)
|
|
378
|
-
|
|
379
353
|
def run_interpreter_exomiser(self, data=None, **options):
|
|
380
354
|
"""
|
|
381
355
|
Run exomiser interpretation analysis.
|
|
@@ -401,62 +375,26 @@ class ClinicalAnalysis(_ParentRestClient):
|
|
|
401
375
|
|
|
402
376
|
return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/exomiser', data=data, **options)
|
|
403
377
|
|
|
404
|
-
def
|
|
405
|
-
"""
|
|
406
|
-
Run TEAM interpretation analysis.
|
|
407
|
-
PATH: /{apiVersion}/analysis/clinical/interpreter/team/run
|
|
408
|
-
|
|
409
|
-
:param dict data: TEAM interpretation analysis params. (REQUIRED)
|
|
410
|
-
:param str study: Study [[organization@]project:]study where study and
|
|
411
|
-
project can be either the ID or UUID.
|
|
412
|
-
:param str job_id: Job ID. It must be a unique string within the
|
|
413
|
-
study. An ID will be autogenerated automatically if not provided.
|
|
414
|
-
:param str job_description: Job description.
|
|
415
|
-
:param str job_depends_on: Comma separated list of existing job IDs
|
|
416
|
-
the job will depend on.
|
|
417
|
-
:param str job_tags: Job tags.
|
|
418
|
-
:param str job_scheduled_start_time: Time when the job is scheduled to
|
|
419
|
-
start.
|
|
420
|
-
:param str job_priority: Priority of the job.
|
|
421
|
-
:param bool job_dry_run: Flag indicating that the job will be executed
|
|
422
|
-
in dry-run mode. In this mode, OpenCGA will validate that all
|
|
423
|
-
parameters and prerequisites are correctly set for successful
|
|
424
|
-
execution, but the job will not actually run.
|
|
425
|
-
"""
|
|
426
|
-
|
|
427
|
-
return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/team', data=data, **options)
|
|
428
|
-
|
|
429
|
-
def run_interpreter_tiering(self, data=None, **options):
|
|
378
|
+
def query_interpreter_rd(self, **options):
|
|
430
379
|
"""
|
|
431
|
-
|
|
432
|
-
PATH: /{apiVersion}/analysis/clinical/interpreter/
|
|
380
|
+
RD interpretation analysis.
|
|
381
|
+
PATH: /{apiVersion}/analysis/clinical/interpreter/rd/query
|
|
433
382
|
|
|
434
|
-
:param
|
|
383
|
+
:param str clinical_analysis_id: Comma separated list of clinical
|
|
384
|
+
analysis IDs or names up to a maximum of 100.
|
|
435
385
|
:param str study: Study [[organization@]project:]study where study and
|
|
436
386
|
project can be either the ID or UUID.
|
|
437
|
-
:param str job_id: Job ID. It must be a unique string within the
|
|
438
|
-
study. An ID will be autogenerated automatically if not provided.
|
|
439
|
-
:param str job_description: Job description.
|
|
440
|
-
:param str job_depends_on: Comma separated list of existing job IDs
|
|
441
|
-
the job will depend on.
|
|
442
|
-
:param str job_tags: Job tags.
|
|
443
|
-
:param str job_scheduled_start_time: Time when the job is scheduled to
|
|
444
|
-
start.
|
|
445
|
-
:param str job_priority: Priority of the job.
|
|
446
|
-
:param bool job_dry_run: Flag indicating that the job will be executed
|
|
447
|
-
in dry-run mode. In this mode, OpenCGA will validate that all
|
|
448
|
-
parameters and prerequisites are correctly set for successful
|
|
449
|
-
execution, but the job will not actually run.
|
|
450
387
|
"""
|
|
451
388
|
|
|
452
|
-
return self.
|
|
389
|
+
return self._get(category='analysis', resource='query', subcategory='clinical/interpreter/rd', **options)
|
|
453
390
|
|
|
454
|
-
def
|
|
391
|
+
def run_interpreter_rd(self, data=None, **options):
|
|
455
392
|
"""
|
|
456
|
-
Run
|
|
457
|
-
PATH: /{apiVersion}/analysis/clinical/interpreter/
|
|
393
|
+
Run clinical interpretation analysis for rare diseases.
|
|
394
|
+
PATH: /{apiVersion}/analysis/clinical/interpreter/rd/run
|
|
458
395
|
|
|
459
|
-
:param dict data:
|
|
396
|
+
:param dict data: Parameters to execute the rare disease
|
|
397
|
+
interpretation analysis. (REQUIRED)
|
|
460
398
|
:param str study: Study [[organization@]project:]study where study and
|
|
461
399
|
project can be either the ID or UUID.
|
|
462
400
|
:param str job_id: Job ID. It must be a unique string within the
|
|
@@ -474,7 +412,7 @@ class ClinicalAnalysis(_ParentRestClient):
|
|
|
474
412
|
execution, but the job will not actually run.
|
|
475
413
|
"""
|
|
476
414
|
|
|
477
|
-
return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/
|
|
415
|
+
return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/rd', data=data, **options)
|
|
478
416
|
|
|
479
417
|
def load(self, data=None, **options):
|
|
480
418
|
"""
|
|
@@ -192,19 +192,6 @@ class DiseasePanel(_ParentRestClient):
|
|
|
192
192
|
|
|
193
193
|
:param str study: Study [[organization@]project:]study where study and
|
|
194
194
|
project can be either the ID or UUID.
|
|
195
|
-
:param str job_id: Job ID. It must be a unique string within the
|
|
196
|
-
study. An ID will be autogenerated automatically if not provided.
|
|
197
|
-
:param str job_depends_on: Comma separated list of existing job IDs
|
|
198
|
-
the job will depend on.
|
|
199
|
-
:param str job_description: Job description.
|
|
200
|
-
:param str job_tags: Job tags.
|
|
201
|
-
:param str job_scheduled_start_time: Time when the job is scheduled to
|
|
202
|
-
start.
|
|
203
|
-
:param str job_priority: Priority of the job.
|
|
204
|
-
:param bool job_dry_run: Flag indicating that the job will be executed
|
|
205
|
-
in dry-run mode. In this mode, OpenCGA will validate that all
|
|
206
|
-
parameters and prerequisites are correctly set for successful
|
|
207
|
-
execution, but the job will not actually run.
|
|
208
195
|
:param dict data: Panel parameters.
|
|
209
196
|
"""
|
|
210
197
|
|
|
@@ -420,7 +420,7 @@ class File(_ParentRestClient):
|
|
|
420
420
|
MICROARRAY_EXPRESSION_TWOCHANNELS_GENEPIX DATAMATRIX_EXPRESSION
|
|
421
421
|
IDLIST IDLIST_RANKED ANNOTATION_GENEVSANNOTATION OTHER_NEWICK
|
|
422
422
|
OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF
|
|
423
|
-
OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE
|
|
423
|
+
OTHER_PED VCF4 CVDB VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE
|
|
424
424
|
REFERENCE_GENOME NONE UNKNOWN']
|
|
425
425
|
:param str checksum: Expected MD5 file checksum.
|
|
426
426
|
:param bool resource: Boolean field indicating whether the file is a
|
|
@@ -105,8 +105,7 @@ class VariantOperation(_ParentRestClient):
|
|
|
105
105
|
in dry-run mode. In this mode, OpenCGA will validate that all
|
|
106
106
|
parameters and prerequisites are correctly set for successful
|
|
107
107
|
execution, but the job will not actually run.
|
|
108
|
-
:param str
|
|
109
|
-
project can be either the ID or UUID.
|
|
108
|
+
:param str project: project.
|
|
110
109
|
:param dict data: Parameters to configure a variant annotation
|
|
111
110
|
extension.
|
|
112
111
|
"""
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
pyxetabase/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
pyxetabase/commons.py,sha256=Okt5DgvSpZW9PUsxoLNpROb0BDrdr_zhgry4X3dQpfk,14308
|
|
3
|
+
pyxetabase/exceptions.py,sha256=GmtDcurD3d_fzaa8AEntF4-sZR4Elgg4Iz3z-UUqjYU,274
|
|
4
|
+
pyxetabase/opencga_client.py,sha256=V-M9i28z3A-Vgw5BnG_Ddm0MUdjQ0_XG38gohGtzavI,15386
|
|
5
|
+
pyxetabase/opencga_config.py,sha256=RK23fextK79S5wq4FaQygwuPnE4p4J9GbjY6hJ0Krxc,7009
|
|
6
|
+
pyxetabase/rest_response.py,sha256=TgwTI2LZFF_jV9-HSawGkF_qZ88n-dxEtIKiFcfPyDk,8635
|
|
7
|
+
pyxetabase/retry.py,sha256=LjViQOaa_GkpDFkcRq9jIS183mE9t4Rq0uls9PV_mfI,2297
|
|
8
|
+
pyxetabase/rest_clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
pyxetabase/rest_clients/_parent_rest_clients.py,sha256=ZL9VxTxS1cRcg-ofrE9QJXYL8mlVkbXpvFvwhofo-uI,4280
|
|
10
|
+
pyxetabase/rest_clients/admin_client.py,sha256=WebfbQVa7CjiaNcp5MqwAkgDTqCfW-v5AgTaxmL5EF0,6927
|
|
11
|
+
pyxetabase/rest_clients/alignment_client.py,sha256=srY1fmjMZCPWlbqhrD2gYVhS2zu7fgPx61tZOqQi4Wc,18509
|
|
12
|
+
pyxetabase/rest_clients/clinical_analysis_client.py,sha256=7JN6mg0QoKaC4yGVr5ujXymh6Dn4Ua4g-9ptb66Fi7A,65708
|
|
13
|
+
pyxetabase/rest_clients/cohort_client.py,sha256=D-su-AFOziztVMI-oAcr5G2Uq_psbLV8Wt0bqZ7zZgI,16324
|
|
14
|
+
pyxetabase/rest_clients/cvdb_client.py,sha256=V7ny5nQEeJqsbALAe7JLyoV6b0XyYF7c9lsn-qK-i8o,143102
|
|
15
|
+
pyxetabase/rest_clients/disease_panel_client.py,sha256=WhYn08O4CjfZN-kDZjZgiJge1cfzlGvGL2OO8zUhW_A,17721
|
|
16
|
+
pyxetabase/rest_clients/family_client.py,sha256=n-kyjnrV09R50JxyiHjFs8rCQkOKFU0Z1O7oVYuaKxg,18586
|
|
17
|
+
pyxetabase/rest_clients/federation_client.py,sha256=p5B8dRq9gj-fdgIfIPhmkT7mNU65cRFtH06kaGtmbWE,5066
|
|
18
|
+
pyxetabase/rest_clients/file_client.py,sha256=fwTWnZtprYNDUBvRetLLe6PTBxW_M8LxN6OlAMGWXC4,33730
|
|
19
|
+
pyxetabase/rest_clients/ga4gh_client.py,sha256=fPnmiblnfpz_zcZPJblGqOFksrdiF3MUer3fv2nwQwk,3506
|
|
20
|
+
pyxetabase/rest_clients/individual_client.py,sha256=C59zr_t7tTxNR7ZBeAPY6fyYl59fE7KY05MaLyXWmXw,23225
|
|
21
|
+
pyxetabase/rest_clients/job_client.py,sha256=amoBW3SL56l6sAPCUB55rZhLR8DcdIIfb5ymOZ9NkLQ,19811
|
|
22
|
+
pyxetabase/rest_clients/meta_client.py,sha256=uExVRfdSKiXteLYe7R3q9x3zCp9mMWWMyPDU56TtoDo,2308
|
|
23
|
+
pyxetabase/rest_clients/organization_client.py,sha256=tnwUTCqJoGiKXMQLMv4ymHA16JxnR5iL2SMLaJDBRs8,9465
|
|
24
|
+
pyxetabase/rest_clients/project_client.py,sha256=M8naPsj47z2ylTrJNU_JyHReKzXrB038PoEtKKrtxmc,5212
|
|
25
|
+
pyxetabase/rest_clients/sample_client.py,sha256=VOsPAhw9HwaEHzzw_5gcVQ1v2xSesvzN3TO4z2opaNo,23621
|
|
26
|
+
pyxetabase/rest_clients/study_client.py,sha256=STtboEiGP-lQ1UptjHxk-ANd3uG6cA6U3srvOO4UQIE,21491
|
|
27
|
+
pyxetabase/rest_clients/user_client.py,sha256=frA7-rMii-yoRyca_Orkj1T80OeEe-zCdWZCHKn1sio,7683
|
|
28
|
+
pyxetabase/rest_clients/variant_client.py,sha256=mmBuVE0JBThJr5zsLGci5nykNcCKyfZXRKl-h3HT9PA,75436
|
|
29
|
+
pyxetabase/rest_clients/variant_operation_client.py,sha256=z_9qEpZEoHF8lLfBq-i3wROnoiNjPjBavqPBepiTIyk,38246
|
|
30
|
+
pyxetabase/rest_clients/workflow_client.py,sha256=QYnyI17aNCjq-uXlguaSj78F0xupeWwmf8uYK1Y5tf4,12482
|
|
31
|
+
pyxetabase-4.0.0.dev35.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
32
|
+
pyxetabase-4.0.0.dev35.dist-info/METADATA,sha256=JZ5oFXMccuHKeBA7TwyQqRD4nPcUQhFXJOgfmaKJPpE,5540
|
|
33
|
+
pyxetabase-4.0.0.dev35.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
34
|
+
pyxetabase-4.0.0.dev35.dist-info/top_level.txt,sha256=0m5pDpBX-lM8QpPl7bTpTQAm4kgu2-nr-pcaEu4Tn_8,11
|
|
35
|
+
pyxetabase-4.0.0.dev35.dist-info/RECORD,,
|
pyopencga/__init__.py
DELETED
|
File without changes
|
pyopencga/commons.py
DELETED
|
@@ -1,347 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
import threading
|
|
3
|
-
from time import sleep
|
|
4
|
-
import warnings
|
|
5
|
-
|
|
6
|
-
import requests
|
|
7
|
-
|
|
8
|
-
from pyopencga.exceptions import OpencgaInvalidToken, OpencgaAuthorisationError
|
|
9
|
-
|
|
10
|
-
try:
|
|
11
|
-
from Queue import Queue
|
|
12
|
-
except ImportError:
|
|
13
|
-
from queue import Queue
|
|
14
|
-
|
|
15
|
-
_CALL_BATCH_SIZE = 2000
|
|
16
|
-
_NUM_THREADS_DEFAULT = 4
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def deprecated(func):
|
|
20
|
-
"""Prints a warning for functions marked as deprecated"""
|
|
21
|
-
def new_func(*args, **kwargs):
|
|
22
|
-
warnings.simplefilter('always', DeprecationWarning) # turn off filter
|
|
23
|
-
warnings.warn('Call to deprecated function "{}".'.format(func.__name__),
|
|
24
|
-
category=DeprecationWarning, stacklevel=2)
|
|
25
|
-
warnings.simplefilter('default', DeprecationWarning) # reset filter
|
|
26
|
-
return func(*args, **kwargs)
|
|
27
|
-
return new_func
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def snake_to_camel_case(text):
|
|
31
|
-
"""Converts snake_case to camelCase"""
|
|
32
|
-
components = text.split('_')
|
|
33
|
-
return components[0] + ''.join(x.title() for x in components[1:])
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def _create_rest_url(host, version, sid, category, resource, subcategory=None, query_id=None,
|
|
37
|
-
second_query_id=None, options=None):
|
|
38
|
-
"""Creates the URL for querying the REST service"""
|
|
39
|
-
|
|
40
|
-
# Creating the basic URL
|
|
41
|
-
url = ('/'.join([host,
|
|
42
|
-
'webservices/rest',
|
|
43
|
-
version,
|
|
44
|
-
category
|
|
45
|
-
]))
|
|
46
|
-
|
|
47
|
-
# If subcategory is queried, query_id can be absent
|
|
48
|
-
if query_id is not None:
|
|
49
|
-
url += '/' + query_id
|
|
50
|
-
|
|
51
|
-
if subcategory is not None:
|
|
52
|
-
url += '/' + subcategory
|
|
53
|
-
|
|
54
|
-
if second_query_id is not None:
|
|
55
|
-
url += '/' + second_query_id
|
|
56
|
-
|
|
57
|
-
url += '/' + resource
|
|
58
|
-
|
|
59
|
-
header = {"Accept-Encoding": "gzip"}
|
|
60
|
-
if sid is not None:
|
|
61
|
-
header['Authorization'] = 'Bearer {}'.format(sid)
|
|
62
|
-
|
|
63
|
-
# Checking optional params
|
|
64
|
-
if options is not None:
|
|
65
|
-
opts = []
|
|
66
|
-
for k, v in options.items():
|
|
67
|
-
k = snake_to_camel_case(k)
|
|
68
|
-
if k == 'debug':
|
|
69
|
-
continue
|
|
70
|
-
if isinstance(v, list):
|
|
71
|
-
opts.append(k + '=' + ','.join(map(str, v)))
|
|
72
|
-
else:
|
|
73
|
-
opts.append(k + '=' + str(v))
|
|
74
|
-
if opts:
|
|
75
|
-
url += '?' + '&'.join(opts)
|
|
76
|
-
return url, header
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
def _fetch(config, sid, category, resource, method, subcategory=None, query_id=None,
|
|
80
|
-
second_query_id=None, data=None, options=None):
|
|
81
|
-
"""Queries the REST service retrieving results until exhaustion or limit"""
|
|
82
|
-
# HERE BE DRAGONS
|
|
83
|
-
final_response = None
|
|
84
|
-
|
|
85
|
-
# Setting up skip and limit default parameters
|
|
86
|
-
call_skip = 0
|
|
87
|
-
call_limit = 1000
|
|
88
|
-
max_limit = None
|
|
89
|
-
if options is None:
|
|
90
|
-
opts = {'skip': call_skip, 'limit': call_limit}
|
|
91
|
-
else:
|
|
92
|
-
opts = options.copy() # Do not modify original data!
|
|
93
|
-
if 'skip' not in opts:
|
|
94
|
-
opts['skip'] = call_skip
|
|
95
|
-
# If 'limit' is specified, a maximum of 'limit' results will be returned
|
|
96
|
-
if 'limit' in opts:
|
|
97
|
-
max_limit = int(opts['limit'])
|
|
98
|
-
# Server must be always queried for results in groups of 1000
|
|
99
|
-
opts['limit'] = call_limit
|
|
100
|
-
|
|
101
|
-
# If there is a query_id, the next variables will be used
|
|
102
|
-
total_id_list = [] # All initial ids
|
|
103
|
-
next_id_list = [] # Ids which should be queried again for more results
|
|
104
|
-
next_id_indexes = [] # Ids position in the final response
|
|
105
|
-
if query_id is not None:
|
|
106
|
-
total_id_list = query_id.split(',')
|
|
107
|
-
|
|
108
|
-
# If some query has more than 'call_limit' results, the server will be
|
|
109
|
-
# queried again to retrieve the next 'call_limit results'
|
|
110
|
-
call = True
|
|
111
|
-
current_query_id = None # Current REST query
|
|
112
|
-
current_id_list = None # Current list of ids
|
|
113
|
-
time_out_counter = 0 # Number of times a query is repeated due to time-out
|
|
114
|
-
while call:
|
|
115
|
-
# Check 'limit' parameter if there is a maximum limit of results
|
|
116
|
-
if max_limit is not None and max_limit <= call_limit:
|
|
117
|
-
opts['limit'] = max_limit
|
|
118
|
-
|
|
119
|
-
# Updating query_id and list of ids to query
|
|
120
|
-
if query_id is not None:
|
|
121
|
-
if current_query_id is None:
|
|
122
|
-
current_query_id = query_id
|
|
123
|
-
current_id_list = total_id_list
|
|
124
|
-
current_id_indexes = range(len(total_id_list))
|
|
125
|
-
else:
|
|
126
|
-
current_query_id = ','.join(next_id_list)
|
|
127
|
-
current_id_list = next_id_list
|
|
128
|
-
current_id_indexes = next_id_indexes
|
|
129
|
-
|
|
130
|
-
# Retrieving url
|
|
131
|
-
url, header = _create_rest_url(host=config.host,
|
|
132
|
-
version=config.version,
|
|
133
|
-
category=category,
|
|
134
|
-
sid=sid,
|
|
135
|
-
subcategory=subcategory,
|
|
136
|
-
query_id=current_query_id,
|
|
137
|
-
second_query_id=second_query_id,
|
|
138
|
-
resource=resource,
|
|
139
|
-
options=opts)
|
|
140
|
-
|
|
141
|
-
# DEBUG param
|
|
142
|
-
if opts is not None and 'debug' in opts and opts['debug']:
|
|
143
|
-
sys.stderr.write(url + '\n')
|
|
144
|
-
|
|
145
|
-
# Getting REST response
|
|
146
|
-
if method == 'get':
|
|
147
|
-
try:
|
|
148
|
-
r = requests.get(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
149
|
-
except requests.exceptions.ConnectionError:
|
|
150
|
-
sleep(1)
|
|
151
|
-
r = requests.get(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
152
|
-
elif method == 'post':
|
|
153
|
-
try:
|
|
154
|
-
r = requests.post(url, json=data, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
155
|
-
except requests.exceptions.ConnectionError:
|
|
156
|
-
sleep(1)
|
|
157
|
-
r = requests.post(url, json=data, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
158
|
-
elif method == 'delete':
|
|
159
|
-
try:
|
|
160
|
-
r = requests.delete(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
161
|
-
except requests.exceptions.ConnectionError:
|
|
162
|
-
sleep(1)
|
|
163
|
-
r = requests.delete(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
|
|
164
|
-
else:
|
|
165
|
-
raise NotImplementedError('method: ' + method + ' not implemented.')
|
|
166
|
-
|
|
167
|
-
if r.status_code == 504: # Gateway Time-out
|
|
168
|
-
if time_out_counter == 99:
|
|
169
|
-
msg = 'Server not responding in time'
|
|
170
|
-
raise requests.ConnectionError(msg)
|
|
171
|
-
time_out_counter += 1
|
|
172
|
-
continue
|
|
173
|
-
time_out_counter = 0
|
|
174
|
-
|
|
175
|
-
if r.status_code == 401:
|
|
176
|
-
raise OpencgaInvalidToken(r.content)
|
|
177
|
-
elif r.status_code == 403:
|
|
178
|
-
raise OpencgaAuthorisationError(r.content)
|
|
179
|
-
elif r.status_code != 200:
|
|
180
|
-
raise Exception(r.content)
|
|
181
|
-
|
|
182
|
-
if r.headers['Content-Type'] == 'application/json':
|
|
183
|
-
try:
|
|
184
|
-
response = r.json()
|
|
185
|
-
|
|
186
|
-
# TODO Remove deprecated response and result in future release. Added for backwards compatibility
|
|
187
|
-
if 'response' in response:
|
|
188
|
-
response['responses'] = response['response']
|
|
189
|
-
for query_result in response['responses']:
|
|
190
|
-
if 'result' in query_result:
|
|
191
|
-
query_result['results'] = query_result['result']
|
|
192
|
-
|
|
193
|
-
except ValueError:
|
|
194
|
-
raise ValueError('Bad JSON format retrieved from server')
|
|
195
|
-
elif r.headers['Content-Type'] == 'application/octet-stream':
|
|
196
|
-
return r.content
|
|
197
|
-
else:
|
|
198
|
-
raise ValueError('Unexpected content type retrieved from server ("{}"): "{}"'.format(
|
|
199
|
-
r.headers['Content-Type'], r.content)
|
|
200
|
-
)
|
|
201
|
-
|
|
202
|
-
# Setting up final_response
|
|
203
|
-
if final_response is None:
|
|
204
|
-
final_response = response
|
|
205
|
-
# Concatenating results
|
|
206
|
-
else:
|
|
207
|
-
if query_id is not None:
|
|
208
|
-
for index, res in enumerate(response['responses']):
|
|
209
|
-
id_index = current_id_indexes[index]
|
|
210
|
-
final_response[id_index]['results'] += res['results']
|
|
211
|
-
else:
|
|
212
|
-
final_response['responses'][0]['results'] += response['responses'][0]['results']
|
|
213
|
-
|
|
214
|
-
if query_id is not None:
|
|
215
|
-
# Checking which ids are completely retrieved
|
|
216
|
-
next_id_list = []
|
|
217
|
-
next_id_indexes = []
|
|
218
|
-
for index, res in enumerate(response['responses']):
|
|
219
|
-
if res['numResults'] == call_limit:
|
|
220
|
-
next_id_list.append(current_id_list[index])
|
|
221
|
-
next_id_indexes.append(current_id_indexes[index])
|
|
222
|
-
# Ending REST calling when there are no more ids to retrieve
|
|
223
|
-
if not next_id_list:
|
|
224
|
-
call = False
|
|
225
|
-
else:
|
|
226
|
-
# Ending REST calling when there are no more results to retrieve
|
|
227
|
-
if response['responses'][0]['numResults'] != call_limit:
|
|
228
|
-
call = False
|
|
229
|
-
|
|
230
|
-
# Skipping the first 'limit' results to retrieve the next ones
|
|
231
|
-
opts['skip'] += call_limit
|
|
232
|
-
|
|
233
|
-
# Subtracting the number of returned results from the maximum goal
|
|
234
|
-
if max_limit is not None:
|
|
235
|
-
max_limit -= call_limit
|
|
236
|
-
# When 'limit' is 0 returns all the results. So, break the loop if 0
|
|
237
|
-
if max_limit == 0:
|
|
238
|
-
break
|
|
239
|
-
|
|
240
|
-
return final_response
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
def _worker(queue, results, config, sid, category, resource, method, subcategory=None,
|
|
244
|
-
second_query_id=None, data=None, options=None):
|
|
245
|
-
|
|
246
|
-
"""Manages the queue system for the threads"""
|
|
247
|
-
while True:
|
|
248
|
-
# Fetching new element from the queue
|
|
249
|
-
index, query_id = queue.get()
|
|
250
|
-
response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
|
|
251
|
-
resource=resource, method=method, data=data, query_id=query_id,
|
|
252
|
-
second_query_id=second_query_id, options=options)
|
|
253
|
-
# Store data in results at correct index
|
|
254
|
-
results[index] = response
|
|
255
|
-
# Signaling to the queue that task has been processed
|
|
256
|
-
queue.task_done()
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def merge_query_responses(query_response_list):
|
|
260
|
-
final_response = query_response_list[0]
|
|
261
|
-
for i, query_response in enumerate(query_response_list):
|
|
262
|
-
if i != 0:
|
|
263
|
-
final_response['events'] += query_response['events']
|
|
264
|
-
final_response['time'] += query_response['time']
|
|
265
|
-
# final_response['responses'] += response['responses']
|
|
266
|
-
|
|
267
|
-
for key in query_response['params']:
|
|
268
|
-
if final_response['params'][key] != query_response['params'][key]:
|
|
269
|
-
final_response['params'][key] += ',' + query_response['params'][key]
|
|
270
|
-
|
|
271
|
-
for j, query_result in enumerate(query_response['responses']):
|
|
272
|
-
if len(final_response['responses'])-1 < j:
|
|
273
|
-
final_response['responses'] += []
|
|
274
|
-
for key in query_result:
|
|
275
|
-
if key not in final_response['responses'][j]:
|
|
276
|
-
final_response['responses'][j][key] = query_result[key]
|
|
277
|
-
else:
|
|
278
|
-
if isinstance(query_result[key], (int, list)):
|
|
279
|
-
final_response['responses'][j][key] += query_result[key]
|
|
280
|
-
return final_response
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
def execute(config, sid, category, resource, method, subcategory=None, query_id=None,
|
|
284
|
-
second_query_id=None, data=None, options=None):
|
|
285
|
-
"""Queries the REST service using multiple threads if needed"""
|
|
286
|
-
|
|
287
|
-
# If query_id is an array, convert to comma-separated string
|
|
288
|
-
if query_id is not None:
|
|
289
|
-
if isinstance(query_id, list):
|
|
290
|
-
query_id = ','.join([str(item) for item in query_id])
|
|
291
|
-
else:
|
|
292
|
-
query_id = str(query_id) # convert to string so we can call this method with int ids
|
|
293
|
-
|
|
294
|
-
# Multithread if the number of queries is greater than _CALL_BATCH_SIZE
|
|
295
|
-
if query_id is None or len(query_id.split(',')) <= _CALL_BATCH_SIZE:
|
|
296
|
-
response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
|
|
297
|
-
resource=resource, method=method, data=data, query_id=query_id,
|
|
298
|
-
second_query_id=second_query_id, options=options)
|
|
299
|
-
return response
|
|
300
|
-
else:
|
|
301
|
-
if options is not None and 'num_threads' in options:
|
|
302
|
-
num_threads = options['num_threads']
|
|
303
|
-
else:
|
|
304
|
-
num_threads = _NUM_THREADS_DEFAULT
|
|
305
|
-
|
|
306
|
-
# Splitting query_id into batches depending on the call batch size
|
|
307
|
-
id_list = query_id.split(',')
|
|
308
|
-
id_batches = [','.join(id_list[x:x + _CALL_BATCH_SIZE])
|
|
309
|
-
for x in range(0, len(id_list), _CALL_BATCH_SIZE)]
|
|
310
|
-
|
|
311
|
-
# Setting up the queue to hold all the id batches
|
|
312
|
-
q = Queue(maxsize=0)
|
|
313
|
-
# Creating a size defined list to store thread results
|
|
314
|
-
res = [''] * len(id_batches)
|
|
315
|
-
|
|
316
|
-
# Setting up the threads
|
|
317
|
-
for thread in range(num_threads):
|
|
318
|
-
t = threading.Thread(target=_worker,
|
|
319
|
-
kwargs={'queue': q,
|
|
320
|
-
'results': res,
|
|
321
|
-
'config': config,
|
|
322
|
-
'sid': sid,
|
|
323
|
-
'category': category,
|
|
324
|
-
'subcategory': subcategory,
|
|
325
|
-
'second_query_id': second_query_id,
|
|
326
|
-
'resource': resource,
|
|
327
|
-
'method': method,
|
|
328
|
-
'data': data,
|
|
329
|
-
'options': options})
|
|
330
|
-
# Setting threads as "daemon" allows main program to exit eventually
|
|
331
|
-
# even if these do not finish correctly
|
|
332
|
-
t.daemon = True
|
|
333
|
-
t.start()
|
|
334
|
-
|
|
335
|
-
# Loading up the queue with index and id batches for each job
|
|
336
|
-
for index, batch in enumerate(id_batches):
|
|
337
|
-
q.put((index, batch)) # Notice this is a tuple
|
|
338
|
-
|
|
339
|
-
# Waiting until the queue has been processed
|
|
340
|
-
q.join()
|
|
341
|
-
|
|
342
|
-
# Joining all the responses into a one final response
|
|
343
|
-
final_query_response = merge_query_responses(res)
|
|
344
|
-
|
|
345
|
-
return final_query_response
|
|
346
|
-
|
|
347
|
-
|
pyopencga/exceptions.py
DELETED