pyxetabase 3.1.0.dev25__py3-none-any.whl → 4.0.0.dev29__py3-none-any.whl

pyxetabase/rest_clients/_parent_rest_clients.py

@@ -1,6 +1,7 @@
-import json
+import os
+import requests
 
-from pyxetabase.commons import execute
+from pyxetabase.commons import execute, _create_rest_url, snake_to_camel_case
 from pyxetabase.rest_response import RestResponse
 from pyxetabase.retry import retry
 
@@ -87,18 +88,23 @@ class _ParentRestClient(object):
     def _post(self, category, resource, data=None, query_id=None, subcategory=None,
               second_query_id=None, **options):
         """Queries the REST service and returns the result"""
-        if data is not None:
-            return self._rest_retry(
-                method='post', category=category, resource=resource, query_id=query_id,
-                subcategory=subcategory, second_query_id=second_query_id,
-                data=data, **options
-            )
+        # Special treatment for the "/{apiVersion}/files/upload" endpoint
+        if category == 'files' and resource == 'upload':
+            response = self._upload(category=category, resource=resource, **options)
+            return RestResponse(response)
         else:
-            return self._rest_retry(
-                method='post', category=category, resource=resource, query_id=query_id,
-                subcategory=subcategory, second_query_id=second_query_id,
-                **options
-            )
+            if data is not None:
+                return self._rest_retry(
+                    method='post', category=category, resource=resource, query_id=query_id,
+                    subcategory=subcategory, second_query_id=second_query_id,
+                    data=data, **options
+                )
+            else:
+                return self._rest_retry(
+                    method='post', category=category, resource=resource, query_id=query_id,
+                    subcategory=subcategory, second_query_id=second_query_id,
+                    **options
+                )
 
     def _delete(self, category, resource, query_id=None, subcategory=None,
                 second_query_id=None, **options):
@@ -108,3 +114,31 @@ class _ParentRestClient(object):
             subcategory=subcategory, second_query_id=second_query_id,
             **options
         )
+
+    def _upload(self, category, resource, **options):
+        """Upload files"""
+
+        # Checking that the parameter file contains the file path to upload
+        if 'file' not in options or not isinstance(options['file'], str):
+            raise ValueError('To upload a file, please specify the file path as the "file" parameter.')
+
+        # Creating URL and headers
+        url, header = _create_rest_url(host=self._cfg.host, version=self._cfg.version, sid=self.token,
+                                       category=category, resource=resource, options=options)
+
+        # Creating data
+        data = {}
+        for k, v in options.items():
+            if k == 'file':  # Param "file" is not included in data
+                continue
+            data[snake_to_camel_case(k)] = v
+
+        # Uploading
+        fpath = os.path.realpath(os.path.expanduser(options['file']))
+        with open(fpath, "rb") as f:
+            fhand = {"file": (fpath, f, "application/octet-stream")}
+            response = requests.post(url, headers=header, files=fhand, data=data or None)
+            if response.status_code != 200:
+                raise Exception(response.content)
+
+        return response.json()

pyxetabase/rest_clients/file_client.py

@@ -420,7 +420,7 @@ class File(_ParentRestClient):
             MICROARRAY_EXPRESSION_TWOCHANNELS_GENEPIX DATAMATRIX_EXPRESSION
             IDLIST IDLIST_RANKED ANNOTATION_GENEVSANNOTATION OTHER_NEWICK
             OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF
-            OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE
+            OTHER_PED VCF4 CVDB VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE
             REFERENCE_GENOME NONE UNKNOWN']
         :param str checksum: Expected MD5 file checksum.
         :param bool resource: Boolean field indicating whether the file is a

pyxetabase/rest_clients/variant_operation_client.py

@@ -105,8 +105,7 @@ class VariantOperation(_ParentRestClient):
             in dry-run mode. In this mode, OpenCGA will validate that all
             parameters and prerequisites are correctly set for successful
             execution, but the job will not actually run.
-        :param str study: Study [[organization@]project:]study where study and
-            project can be either the ID or UUID.
+        :param str project: project.
         :param dict data: Parameters to configure a variant annotation
             extension.
         """
@@ -628,33 +627,6 @@ class VariantOperation(_ParentRestClient):
 
         return self._post(category='operation', resource='secondaryIndex', subcategory='variant', data=data, **options)
 
-    def delete_variant_secondary_index(self, **options):
-        """
-        Remove a secondary index from the search engine for a specific set of
-            samples.
-        PATH: /{apiVersion}/operation/variant/secondaryIndex/delete
-
-        :param str job_id: Job ID. It must be a unique string within the
-            study. An ID will be autogenerated automatically if not provided.
-        :param str job_description: Job description.
-        :param str job_depends_on: Comma separated list of existing job IDs
-            the job will depend on.
-        :param str job_tags: Job tags.
-        :param str job_scheduled_start_time: Time when the job is scheduled to
-            start.
-        :param str job_priority: Priority of the job.
-        :param bool job_dry_run: Flag indicating that the job will be executed
-            in dry-run mode. In this mode, OpenCGA will validate that all
-            parameters and prerequisites are correctly set for successful
-            execution, but the job will not actually run.
-        :param str study: Study [[organization@]project:]study where study and
-            project can be either the ID or UUID.
-        :param str samples: Samples to remove. Needs to provide all the
-            samples in the secondary index.
-        """
-
-        return self._delete(category='operation', resource='delete', subcategory='variant/secondaryIndex', **options)
-
     def setup_variant(self, data=None, **options):
         """
         Execute Variant Setup to allow using the variant engine. This setup is

{pyxetabase-3.1.0.dev25.dist-info → pyxetabase-4.0.0.dev29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyxetabase
-Version: 3.1.0.dev25
+Version: 4.0.0.dev29
 Summary: A REST client for OpenCGA enterprise REST web services
 Author: Daniel Perez-Gil
 Author-email: daniel.perez@zettagenomics.com

pyxetabase-4.0.0.dev29.dist-info/RECORD

@@ -0,0 +1,35 @@
+pyxetabase/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyxetabase/commons.py,sha256=Okt5DgvSpZW9PUsxoLNpROb0BDrdr_zhgry4X3dQpfk,14308
+pyxetabase/exceptions.py,sha256=GmtDcurD3d_fzaa8AEntF4-sZR4Elgg4Iz3z-UUqjYU,274
+pyxetabase/opencga_client.py,sha256=V-M9i28z3A-Vgw5BnG_Ddm0MUdjQ0_XG38gohGtzavI,15386
+pyxetabase/opencga_config.py,sha256=RK23fextK79S5wq4FaQygwuPnE4p4J9GbjY6hJ0Krxc,7009
+pyxetabase/rest_response.py,sha256=TgwTI2LZFF_jV9-HSawGkF_qZ88n-dxEtIKiFcfPyDk,8635
+pyxetabase/retry.py,sha256=LjViQOaa_GkpDFkcRq9jIS183mE9t4Rq0uls9PV_mfI,2297
+pyxetabase/rest_clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyxetabase/rest_clients/_parent_rest_clients.py,sha256=OuCnmaziUCENpmpA3CzKJGEhMY_Z-7Q_rlkH75QDi7I,5854
+pyxetabase/rest_clients/admin_client.py,sha256=WebfbQVa7CjiaNcp5MqwAkgDTqCfW-v5AgTaxmL5EF0,6927
+pyxetabase/rest_clients/alignment_client.py,sha256=srY1fmjMZCPWlbqhrD2gYVhS2zu7fgPx61tZOqQi4Wc,18509
+pyxetabase/rest_clients/clinical_analysis_client.py,sha256=j2LmLT8wg3C9srRMB2mksEVFnqN22Anb7HxZCbdF50c,69126
+pyxetabase/rest_clients/cohort_client.py,sha256=D-su-AFOziztVMI-oAcr5G2Uq_psbLV8Wt0bqZ7zZgI,16324
+pyxetabase/rest_clients/cvdb_client.py,sha256=V7ny5nQEeJqsbALAe7JLyoV6b0XyYF7c9lsn-qK-i8o,143102
+pyxetabase/rest_clients/disease_panel_client.py,sha256=GAYRyAfoB_VunskXOhAzMNbsrFupb6M2QbRK3x1Y5vo,18513
+pyxetabase/rest_clients/family_client.py,sha256=n-kyjnrV09R50JxyiHjFs8rCQkOKFU0Z1O7oVYuaKxg,18586
+pyxetabase/rest_clients/federation_client.py,sha256=p5B8dRq9gj-fdgIfIPhmkT7mNU65cRFtH06kaGtmbWE,5066
+pyxetabase/rest_clients/file_client.py,sha256=fwTWnZtprYNDUBvRetLLe6PTBxW_M8LxN6OlAMGWXC4,33730
+pyxetabase/rest_clients/ga4gh_client.py,sha256=fPnmiblnfpz_zcZPJblGqOFksrdiF3MUer3fv2nwQwk,3506
+pyxetabase/rest_clients/individual_client.py,sha256=C59zr_t7tTxNR7ZBeAPY6fyYl59fE7KY05MaLyXWmXw,23225
+pyxetabase/rest_clients/job_client.py,sha256=amoBW3SL56l6sAPCUB55rZhLR8DcdIIfb5ymOZ9NkLQ,19811
+pyxetabase/rest_clients/meta_client.py,sha256=uExVRfdSKiXteLYe7R3q9x3zCp9mMWWMyPDU56TtoDo,2308
+pyxetabase/rest_clients/organization_client.py,sha256=tnwUTCqJoGiKXMQLMv4ymHA16JxnR5iL2SMLaJDBRs8,9465
+pyxetabase/rest_clients/project_client.py,sha256=M8naPsj47z2ylTrJNU_JyHReKzXrB038PoEtKKrtxmc,5212
+pyxetabase/rest_clients/sample_client.py,sha256=VOsPAhw9HwaEHzzw_5gcVQ1v2xSesvzN3TO4z2opaNo,23621
+pyxetabase/rest_clients/study_client.py,sha256=STtboEiGP-lQ1UptjHxk-ANd3uG6cA6U3srvOO4UQIE,21491
+pyxetabase/rest_clients/user_client.py,sha256=frA7-rMii-yoRyca_Orkj1T80OeEe-zCdWZCHKn1sio,7683
+pyxetabase/rest_clients/variant_client.py,sha256=mmBuVE0JBThJr5zsLGci5nykNcCKyfZXRKl-h3HT9PA,75436
+pyxetabase/rest_clients/variant_operation_client.py,sha256=ceZ2-ii0D1oCDMRz_EURwUf_VjtsV5lzJuY-ZwG1b6Q,36840
+pyxetabase/rest_clients/workflow_client.py,sha256=QYnyI17aNCjq-uXlguaSj78F0xupeWwmf8uYK1Y5tf4,12482
+pyxetabase-4.0.0.dev29.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+pyxetabase-4.0.0.dev29.dist-info/METADATA,sha256=TL6HOzwe313tz-0JeNMPB2aGt2WsY0qniQ1Xl2oxiUc,5540
+pyxetabase-4.0.0.dev29.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pyxetabase-4.0.0.dev29.dist-info/top_level.txt,sha256=0m5pDpBX-lM8QpPl7bTpTQAm4kgu2-nr-pcaEu4Tn_8,11
+pyxetabase-4.0.0.dev29.dist-info/RECORD,,

pyopencga/commons.py

@@ -1,347 +0,0 @@
-import sys
-import threading
-from time import sleep
-import warnings
-
-import requests
-
-from pyopencga.exceptions import OpencgaInvalidToken, OpencgaAuthorisationError
-
-try:
-    from Queue import Queue
-except ImportError:
-    from queue import Queue
-
-_CALL_BATCH_SIZE = 2000
-_NUM_THREADS_DEFAULT = 4
-
-
-def deprecated(func):
-    """Prints a warning for functions marked as deprecated"""
-    def new_func(*args, **kwargs):
-        warnings.simplefilter('always', DeprecationWarning)  # turn off filter
-        warnings.warn('Call to deprecated function "{}".'.format(func.__name__),
-                      category=DeprecationWarning, stacklevel=2)
-        warnings.simplefilter('default', DeprecationWarning)  # reset filter
-        return func(*args, **kwargs)
-    return new_func
-
-
-def snake_to_camel_case(text):
-    """Converts snake_case to camelCase"""
-    components = text.split('_')
-    return components[0] + ''.join(x.title() for x in components[1:])
-
-
-def _create_rest_url(host, version, sid, category, resource, subcategory=None, query_id=None,
-                     second_query_id=None, options=None):
-    """Creates the URL for querying the REST service"""
-
-    # Creating the basic URL
-    url = ('/'.join([host,
-                     'webservices/rest',
-                     version,
-                     category
-                     ]))
-
-    # If subcategory is queried, query_id can be absent
-    if query_id is not None:
-        url += '/' + query_id
-
-    if subcategory is not None:
-        url += '/' + subcategory
-
-    if second_query_id is not None:
-        url += '/' + second_query_id
-
-    url += '/' + resource
-
-    header = {"Accept-Encoding": "gzip"}
-    if sid is not None:
-        header['Authorization'] = 'Bearer {}'.format(sid)
-
-    # Checking optional params
-    if options is not None:
-        opts = []
-        for k, v in options.items():
-            k = snake_to_camel_case(k)
-            if k == 'debug':
-                continue
-            if isinstance(v, list):
-                opts.append(k + '=' + ','.join(map(str, v)))
-            else:
-                opts.append(k + '=' + str(v))
-        if opts:
-            url += '?' + '&'.join(opts)
-    return url, header
-
-
-def _fetch(config, sid, category, resource, method, subcategory=None, query_id=None,
-           second_query_id=None, data=None, options=None):
-    """Queries the REST service retrieving results until exhaustion or limit"""
-    # HERE BE DRAGONS
-    final_response = None
-
-    # Setting up skip and limit default parameters
-    call_skip = 0
-    call_limit = 1000
-    max_limit = None
-    if options is None:
-        opts = {'skip': call_skip, 'limit': call_limit}
-    else:
-        opts = options.copy()  # Do not modify original data!
-        if 'skip' not in opts:
-            opts['skip'] = call_skip
-        # If 'limit' is specified, a maximum of 'limit' results will be returned
-        if 'limit' in opts:
-            max_limit = int(opts['limit'])
-        # Server must be always queried for results in groups of 1000
-        opts['limit'] = call_limit
-
-    # If there is a query_id, the next variables will be used
-    total_id_list = []  # All initial ids
-    next_id_list = []  # Ids which should be queried again for more results
-    next_id_indexes = []  # Ids position in the final response
-    if query_id is not None:
-        total_id_list = query_id.split(',')
-
-    # If some query has more than 'call_limit' results, the server will be
-    # queried again to retrieve the next 'call_limit results'
-    call = True
-    current_query_id = None  # Current REST query
-    current_id_list = None  # Current list of ids
-    time_out_counter = 0  # Number of times a query is repeated due to time-out
-    while call:
-        # Check 'limit' parameter if there is a maximum limit of results
-        if max_limit is not None and max_limit <= call_limit:
-            opts['limit'] = max_limit
-
-        # Updating query_id and list of ids to query
-        if query_id is not None:
-            if current_query_id is None:
-                current_query_id = query_id
-                current_id_list = total_id_list
-                current_id_indexes = range(len(total_id_list))
-            else:
-                current_query_id = ','.join(next_id_list)
-                current_id_list = next_id_list
-                current_id_indexes = next_id_indexes
-
-        # Retrieving url
-        url, header = _create_rest_url(host=config.host,
-                                       version=config.version,
-                                       category=category,
-                                       sid=sid,
-                                       subcategory=subcategory,
-                                       query_id=current_query_id,
-                                       second_query_id=second_query_id,
-                                       resource=resource,
-                                       options=opts)
-
-        # DEBUG param
-        if opts is not None and 'debug' in opts and opts['debug']:
-            sys.stderr.write(url + '\n')
-
-        # Getting REST response
-        if method == 'get':
-            try:
-                r = requests.get(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-            except requests.exceptions.ConnectionError:
-                sleep(1)
-                r = requests.get(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-        elif method == 'post':
-            try:
-                r = requests.post(url, json=data, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-            except requests.exceptions.ConnectionError:
-                sleep(1)
-                r = requests.post(url, json=data, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-        elif method == 'delete':
-            try:
-                r = requests.delete(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-            except requests.exceptions.ConnectionError:
-                sleep(1)
-                r = requests.delete(url, headers=header, cookies=config.cookies, verify=not config.tlsAllowInvalidCertificates)
-        else:
-            raise NotImplementedError('method: ' + method + ' not implemented.')
-
-        if r.status_code == 504:  # Gateway Time-out
-            if time_out_counter == 99:
-                msg = 'Server not responding in time'
-                raise requests.ConnectionError(msg)
-            time_out_counter += 1
-            continue
-        time_out_counter = 0
-
-        if r.status_code == 401:
-            raise OpencgaInvalidToken(r.content)
-        elif r.status_code == 403:
-            raise OpencgaAuthorisationError(r.content)
-        elif r.status_code != 200:
-            raise Exception(r.content)
-
-        if r.headers['Content-Type'] == 'application/json':
-            try:
-                response = r.json()
-
-                # TODO Remove deprecated response and result in future release. Added for backwards compatibility
-                if 'response' in response:
-                    response['responses'] = response['response']
-                for query_result in response['responses']:
-                    if 'result' in query_result:
-                        query_result['results'] = query_result['result']
-
-            except ValueError:
-                raise ValueError('Bad JSON format retrieved from server')
-        elif r.headers['Content-Type'] == 'application/octet-stream':
-            return r.content
-        else:
-            raise ValueError('Unexpected content type retrieved from server ("{}"): "{}"'.format(
-                r.headers['Content-Type'], r.content)
-            )
-
-        # Setting up final_response
-        if final_response is None:
-            final_response = response
-        # Concatenating results
-        else:
-            if query_id is not None:
-                for index, res in enumerate(response['responses']):
-                    id_index = current_id_indexes[index]
-                    final_response[id_index]['results'] += res['results']
-            else:
-                final_response['responses'][0]['results'] += response['responses'][0]['results']
-
-        if query_id is not None:
-            # Checking which ids are completely retrieved
-            next_id_list = []
-            next_id_indexes = []
-            for index, res in enumerate(response['responses']):
-                if res['numResults'] == call_limit:
-                    next_id_list.append(current_id_list[index])
-                    next_id_indexes.append(current_id_indexes[index])
-            # Ending REST calling when there are no more ids to retrieve
-            if not next_id_list:
-                call = False
-        else:
-            # Ending REST calling when there are no more results to retrieve
-            if response['responses'][0]['numResults'] != call_limit:
-                call = False
-
-        # Skipping the first 'limit' results to retrieve the next ones
-        opts['skip'] += call_limit
-
-        # Subtracting the number of returned results from the maximum goal
-        if max_limit is not None:
-            max_limit -= call_limit
-            # When 'limit' is 0 returns all the results. So, break the loop if 0
-            if max_limit == 0:
-                break
-
-    return final_response
-
-
-def _worker(queue, results, config, sid, category, resource, method, subcategory=None,
-            second_query_id=None, data=None, options=None):
-
-    """Manages the queue system for the threads"""
-    while True:
-        # Fetching new element from the queue
-        index, query_id = queue.get()
-        response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
-                          resource=resource, method=method, data=data, query_id=query_id,
-                          second_query_id=second_query_id, options=options)
-        # Store data in results at correct index
-        results[index] = response
-        # Signaling to the queue that task has been processed
-        queue.task_done()
-
-
-def merge_query_responses(query_response_list):
-    final_response = query_response_list[0]
-    for i, query_response in enumerate(query_response_list):
-        if i != 0:
-            final_response['events'] += query_response['events']
-            final_response['time'] += query_response['time']
-            # final_response['responses'] += response['responses']
-
-            for key in query_response['params']:
-                if final_response['params'][key] != query_response['params'][key]:
-                    final_response['params'][key] += ',' + query_response['params'][key]
-
-            for j, query_result in enumerate(query_response['responses']):
-                if len(final_response['responses'])-1 < j:
-                    final_response['responses'] += []
-                for key in query_result:
-                    if key not in final_response['responses'][j]:
-                        final_response['responses'][j][key] = query_result[key]
-                    else:
-                        if isinstance(query_result[key], (int, list)):
-                            final_response['responses'][j][key] += query_result[key]
-    return final_response
-
-
-def execute(config, sid, category, resource, method, subcategory=None, query_id=None,
-            second_query_id=None, data=None, options=None):
-    """Queries the REST service using multiple threads if needed"""
-
-    # If query_id is an array, convert to comma-separated string
-    if query_id is not None:
-        if isinstance(query_id, list):
-            query_id = ','.join([str(item) for item in query_id])
-        else:
-            query_id = str(query_id)  # convert to string so we can call this method with int ids
-
-    # Multithread if the number of queries is greater than _CALL_BATCH_SIZE
-    if query_id is None or len(query_id.split(',')) <= _CALL_BATCH_SIZE:
-        response = _fetch(config=config, sid=sid, category=category, subcategory=subcategory,
-                          resource=resource, method=method, data=data, query_id=query_id,
-                          second_query_id=second_query_id, options=options)
-        return response
-    else:
-        if options is not None and 'num_threads' in options:
-            num_threads = options['num_threads']
-        else:
-            num_threads = _NUM_THREADS_DEFAULT
-
-        # Splitting query_id into batches depending on the call batch size
-        id_list = query_id.split(',')
-        id_batches = [','.join(id_list[x:x + _CALL_BATCH_SIZE])
-                      for x in range(0, len(id_list), _CALL_BATCH_SIZE)]
-
-        # Setting up the queue to hold all the id batches
-        q = Queue(maxsize=0)
-        # Creating a size defined list to store thread results
-        res = [''] * len(id_batches)
-
-        # Setting up the threads
-        for thread in range(num_threads):
-            t = threading.Thread(target=_worker,
-                                 kwargs={'queue': q,
-                                         'results': res,
-                                         'config': config,
-                                         'sid': sid,
-                                         'category': category,
-                                         'subcategory': subcategory,
-                                         'second_query_id': second_query_id,
-                                         'resource': resource,
-                                         'method': method,
-                                         'data': data,
-                                         'options': options})
-            # Setting threads as "daemon" allows main program to exit eventually
-            # even if these do not finish correctly
-            t.daemon = True
-            t.start()
-
-        # Loading up the queue with index and id batches for each job
-        for index, batch in enumerate(id_batches):
-            q.put((index, batch))  # Notice this is a tuple
-
-        # Waiting until the queue has been processed
-        q.join()
-
-    # Joining all the responses into a one final response
-    final_query_response = merge_query_responses(res)
-
-    return final_query_response
-
-

pyopencga/exceptions.py

@@ -1,8 +0,0 @@
-class OpencgaAuthorisationError(Exception):
-    def __init__(self, message):
-        super(OpencgaAuthorisationError, self).__init__(message)
-
-
-class OpencgaInvalidToken(Exception):
-    def __init__(self, message):
-        super(OpencgaInvalidToken, self).__init__(message)