pyxetabase 3.1.0.dev0__py3-none-any.whl

pyxetabase/opencga_client.py

@@ -0,0 +1,344 @@
+import getpass
+import time
+import sys
+import re
+if sys.version_info >= (3, 8):
+    from importlib.metadata import version
+else:
+    from importlib_metadata import version
+
+from pyxetabase.opencga_config import ClientConfiguration
+from pyxetabase.rest_clients.admin_client import Admin
+from pyxetabase.rest_clients.alignment_client import Alignment
+from pyxetabase.rest_clients.clinical_analysis_client import ClinicalAnalysis
+from pyxetabase.rest_clients.cohort_client import Cohort
+from pyxetabase.rest_clients.family_client import Family
+from pyxetabase.rest_clients.file_client import File
+from pyxetabase.rest_clients.ga4gh_client import GA4GH
+from pyxetabase.rest_clients.individual_client import Individual
+from pyxetabase.rest_clients.job_client import Job
+from pyxetabase.rest_clients.meta_client import Meta
+from pyxetabase.rest_clients.disease_panel_client import DiseasePanel
+from pyxetabase.rest_clients.project_client import Project
+from pyxetabase.rest_clients.sample_client import Sample
+from pyxetabase.rest_clients.study_client import Study
+from pyxetabase.rest_clients.variant_operation_client import VariantOperation
+from pyxetabase.rest_clients.user_client import User
+from pyxetabase.rest_clients.variant_client import Variant
+from pyxetabase.rest_clients.organization_client import Organization
+from pyxetabase.rest_clients.workflow_client import Workflow
+from pyxetabase.rest_clients.federation_client import Federation
+from pyxetabase.rest_clients.cvdb_client import CVDB
+
+
+class OpencgaClient(object):
+    def __init__(self, configuration, token=None, refresh_token=None, on_retry=None, auto_refresh=True):
+        """
+        :param on_retry: callback to be called with client retries an operation.
+            It must accept parameters: client, exc_type, exc_val, exc_tb, call
+        """
+
+        if not isinstance(configuration, ClientConfiguration):
+            raise ValueError('Expected ClientConfiguration instance')
+
+        self.configuration = configuration
+        self.auto_refresh = auto_refresh
+        self.on_retry = on_retry
+        self.clients = []
+        self._login_handler = None
+        self.token = token
+        self.refresh_token = refresh_token
+        self._create_clients()
+        self._check_versions()
+        if self.configuration.sso_login:
+            self._sso_login()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.logout()
+
+    def _check_versions(self):
+        # Getting client and server versions
+        client_version = version("pyopencga")
+        server_version = self.meta.about().get_result(0)['Version'].split('-')[0]
+
+        ansi_reset = "\033[0m"
+        ansi_red = "\033[31m"
+        ansi_yellow = "\033[33m"
+        if tuple(server_version.split('.')[:2]) < tuple(client_version.split('.')[:2]):
+            msg = '[WARNING]: Client version ({}) is higher than server version ({}).' \
+                  ' Some client features may not be implemented in the server.\n'.format(client_version, server_version)
+            sys.stdout.write('{}{}{}'.format(ansi_red, msg, ansi_reset))
+        elif tuple(server_version.split('.')[:2]) > tuple(client_version.split('.')[:2]):
+            msg = '[INFO]: Client version ({}) is lower than server version ({}).' \
+                   ' Some client features may not work as intended.\n'.format(client_version, server_version)
+            sys.stdout.write('{}{}{}'.format(ansi_yellow, msg, ansi_reset))
+
+    def _create_clients(self):
+        self.organizations = Organization(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.users = User(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.projects = Project(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.studies = Study(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.files = File(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.jobs = Job(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.samples = Sample(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.individuals = Individual(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.families = Family(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.cohorts = Cohort(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.disease_panels = DiseasePanel(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.alignments = Alignment(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.variants = Variant(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.clinical = ClinicalAnalysis(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.operations = VariantOperation(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.variant_operations = self.operations  # DEPRECATED: use 'self.operations'
+        self.meta = Meta(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.ga4gh = GA4GH(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.admin = Admin(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.workflows = Workflow(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.federations = Federation(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+        self.cvdb = CVDB(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh)
+
+        self.clients = [
+            self.organizations, self.users, self.projects, self.studies, self.files, self.jobs,
+            self.samples, self.individuals, self.families, self.cohorts,
+            self.disease_panels, self.alignments, self.variants, self.clinical, self.operations,
+            self.variant_operations, self.meta, self.ga4gh, self.admin, self.workflows, self.federations, self.cvdb
+        ]
+
+        for client in self.clients:
+            client.on_retry = self.on_retry
+
+    def _make_login_handler(self, user=None, password=None, organization=None):
+        """
+        Returns a closure that performs the log-in. This will be called on retries
+        if the current session ever expires.
+        The reason for using a closure and not a normal function is that a normal
+        function would require storing the password in a field. It is more secure
+        not to do so. This way, the password stored in the closure is inaccessible
+        to other code
+        """
+        def login_handler(refresh=False):
+            if refresh:
+                data = {'refreshToken': self.refresh_token}
+            else:
+                data = {'user': user, 'password': password}
+            if organization:
+                data.update({'organization': organization})
+            tokens = User(self.configuration).login(data=data).get_result(0)
+            self.token = tokens['token']
+            self.refresh_token = tokens['refreshToken']
+            return self.token, self.refresh_token
+        return login_handler
+
+    def login(self, user=None, password=None, organization=None):
+        if user is not None and password is None:
+            password = getpass.getpass()
+        if not (user and password):
+            raise ValueError('User and password required')
+
+        self._login_handler = self._make_login_handler(user=user, password=password, organization=organization)
+        self._login_handler()
+        for client in self.clients:
+            client.token = self.token
+            client.refresh_token = self.refresh_token
+            client.login_handler = self._login_handler
+
+    def _sso_login(self):
+        # Getting token and refresh token from configuration
+        self.token = self.configuration.token
+        self.refresh_token = self.configuration.token
+
+        self._login_handler = self._make_login_handler()
+        for client in self.clients:
+            client.token = self.token
+            client.refresh_token = self.refresh_token
+            client.login_handler = self._login_handler
+
+    def logout(self):
+        self.token = None
+        self.refresh_token = None
+        for client in self.clients:
+            client.token = self.token
+            client.refresh_token = self.refresh_token
+
+    def wait_for_job(self, response=None, study_id=None, job_id=None, retry_seconds=10):
+        if response is not None:
+            study_id = response['results'][0]['study']['id']
+            job_id = response['results'][0]['id']
+
+        if response is None and (study_id is None or job_id is None):
+            raise ValueError('Argument "response" or arguments "study" and "job_id" must be provided')
+
+        if len(job_id.split(',')) > 1:
+            raise ValueError('Only one job ID is allowed')
+
+        retry_seconds = retry_seconds if retry_seconds >= 10 else 10
+        while True:
+            job_info = self.jobs.info(study=study_id, jobs=job_id).get_result(0)
+            if job_info['internal']['status']['id'] in ['ERROR', 'ABORTED']:
+                raise ValueError('{} ({}): {}'.format(
+                    job_info['internal']['status']['id'],
+                    job_info['internal']['status']['date'],
+                    job_info['internal']['status']['description']
+                ))
+            elif job_info['internal']['status']['id'] in ['DONE']:
+                break
+            time.sleep(retry_seconds)
+
+    def _get_help_info(self, client_name=None, parameters=False):
+        info = []
+        for client in self.clients:
+            # Name
+            cls_name = type(client).__name__
+            client_method = re.sub(r'(?<!^)(?=[A-Z])', '_', cls_name).lower() \
+                if cls_name != 'GA4GH' else cls_name.lower()
+            client_method = 'get_' + client_method + '_client'
+
+            if client_name is not None and client_name != cls_name:
+                continue
+
+            # Description and path
+            class_docstring = client.__doc__
+            cls_desc = re.findall('(This class contains methods .+)\n', class_docstring)[0]
+            cls_path = re.findall('PATH: (.+)\n', class_docstring)[0]
+
+            # Methods
+            methods = []
+            method_names = [method_name for method_name in dir(client)
+                            if callable(getattr(client, method_name))
+                            and not method_name.startswith('_') and method_name != 'login_handler']
+            for method_name in method_names:
+                if client_name is None:
+                    continue
+                method_docstring = getattr(client, method_name).__doc__
+                desc = re.findall('(.+)\n +PATH', method_docstring, re.DOTALL)
+                desc = re.sub(' +', ' ', desc[0].replace('\n', ' ').strip())
+                path = re.findall('PATH: (.+)\n', method_docstring)[0]
+
+                args = []
+                arguments = re.findall(
+                    ' +:param (.+)', method_docstring, re.DOTALL
+                )
+                if arguments and parameters:
+                    arguments = arguments[0].replace('\n', ' ').strip()
+                    arguments = re.sub(' +', ' ', arguments)
+                    arguments = arguments.split(' :param ')
+                    for parameter in arguments:
+                        param_info = parameter.split(' ', 2)
+                        args.append({
+                            'name': param_info[1].rstrip(':'),
+                            'type': param_info[0],
+                            'desc': param_info[2]
+                        })
+                methods.append({
+                    'name': method_name,
+                    'desc': desc,
+                    'path': path,
+                    'params': args
+                })
+
+            info.append(
+                {'class_name': cls_name, 'client_method': client_method,
+                 'desc': cls_desc, 'path': cls_path, 'methods': methods}
+            )
+        return info
+
+    def help(self, client_name=None, show_parameters=False):
+        help_txt = []
+
+        info = self._get_help_info(client_name, show_parameters)
+        if client_name is None:
+            help_txt += ['Available clients:']
+            for client in info:
+                txt = '{}- {}: {} ({}). USAGE: opencga_client.{}()'
+                help_txt += [txt.format(
+                    ' '*4, client['class_name'], client['desc'],
+                    client['path'], client['client_method']
+                )]
+        else:
+            for client in info:
+                help_txt += ['{}: {} ({}). USAGE: opencga_client.{}()'.format(
+                    client['class_name'], client['desc'], client['path'],
+                    client['client_method']
+                )]
+                help_txt += ['{}Available methods:'.format(' '*4)]
+                for method in client['methods']:
+                    help_txt += ['{}- {}: {} ({})'.format(
+                        ' '*8, method['name'], method['desc'], method['path']
+                    )]
+                    if not show_parameters:
+                        continue
+                    for param in method['params']:
+                        help_txt += ['{}* {} ({}): {}'.format(
+                            ' ' * 12, param['name'], param['type'],
+                            param['desc']
+                        )]
+        sys.stdout.write('\n'.join(help_txt) + '\n')
+
+    def get_organization_client(self):
+        return self.organizations
+
+    def get_user_client(self):
+        return self.users
+
+    def get_project_client(self):
+        return self.projects
+
+    def get_study_client(self):
+        return self.studies
+
+    def get_file_client(self):
+        return self.files
+
+    def get_job_client(self):
+        return self.jobs
+
+    def get_sample_client(self):
+        return self.samples
+
+    def get_individual_client(self):
+        return self.individuals
+
+    def get_family_client(self):
+        return self.families
+
+    def get_cohort_client(self):
+        return self.cohorts
+
+    def get_disease_panel_client(self):
+        return self.disease_panels
+
+    def get_alignment_client(self):
+        return self.alignments
+
+    def get_variant_client(self):
+        return self.variants
+
+    def get_clinical_client(self):
+        return self.clinical
+
+    def get_operations_client(self):
+        return self.operations
+
+    def get_variant_operation_client(self):
+        return self.variant_operations
+
+    def get_meta_client(self):
+        return self.meta
+
+    def get_ga4gh_client(self):
+        return self.ga4gh
+
+    def get_admin_client(self):
+        return self.admin
+
+    def get_workflows_client(self):
+        return self.workflows
+
+    def get_federations_client(self):
+        return self.federations
+
+    def get_cvdb_client(self):
+        return self.cvdb

pyxetabase/opencga_config.py

@@ -0,0 +1,160 @@
+import os
+import json
+import requests
+import yaml
+
+
+class ClientConfiguration(object):
+    """PyOpenCGA configuration"""
+    
+    def __init__(self, config_input):
+        """
+        :param dict or str config_input: a dict, or a yaml/json file containing an OpenCGA valid client configuration.
+        """
+
+        if isinstance(config_input, dict):
+            self._config = config_input
+        else:
+            self._config = self._get_dictionary_from_file(config_input)
+
+        self.get_sso_login_info()
+
+        self._validate_configuration()
+
+    @staticmethod
+    def _get_dictionary_from_file(config_fpath):
+        config_fhand = open(config_fpath, 'r')
+
+        if config_fpath.endswith('.yml') or config_fpath.endswith('.yaml'):
+            config_dict = yaml.safe_load(config_fhand)
+        elif config_fpath.endswith('.json'):
+            config_dict = json.loads(config_fhand.read())
+        else:
+            raise NotImplementedError('Input must be a dictionary, a yaml file (.yml or .yaml) or a json file (.json)')
+
+        config_fhand.close()
+
+        return config_dict
+
+    def get_sso_login_info(self):
+        if (('sso_login' in self._config and self._config['sso_login']) or
+                ('cookies' in self._config and self._config['cookies'])):
+            python_session_fhand = open(os.path.expanduser("~/.opencga/session.json"), 'r')
+            session_info = json.loads(python_session_fhand.read())
+            self._config['sso_login'] = True
+            self._config['cookies'] = session_info['cookies']
+            self._config['token'] = session_info['token']
+
+    def _validate_configuration(self):
+        if self._config is None:
+            raise ValueError('Missing configuration dictionary.')
+
+        if 'rest' not in self._config:
+            raise ValueError('Missing configuration field "rest".')
+
+        if 'host' not in self._config['rest'] or not self._config['rest']['host']:
+            raise ValueError('Missing configuration field "host".')
+
+        self._validate_host(self._config['rest']['host'])
+
+    @staticmethod
+    def _validate_host(host):
+        try:
+            r = requests.head(host, timeout=2)
+            if r.status_code == 302:
+                return
+        except requests.ConnectionError:
+            raise Exception('Unreachable host "{}"'.format(host))
+
+    @property
+    def configuration(self):
+        return self._config
+
+    @property
+    def host(self):
+        return self._config['rest']['host']
+
+    @host.setter
+    def host(self, host):
+        self._config['rest']['host'] = host
+        self._validate_host(host)
+
+    @property
+    def version(self):
+        return self._config['version'] if 'version' in self._config else 'v2'
+
+    @version.setter
+    def version(self, version):
+        self._config['version'] = version
+
+    @property
+    def sso_login(self):
+        if 'sso_login' in self._config and self._config['sso_login']:
+            return self._config['sso_login']
+        else:
+            return False
+
+    @sso_login.setter
+    def sso_login(self, sso_login):
+        if isinstance(str, sso_login):
+            self._config['sso_login'] = sso_login.lower() == 'true'
+        if isinstance(bool, sso_login):
+            self._config['sso_login'] = sso_login
+
+    @property
+    def cookies(self):
+        if 'cookies' in self._config and self._config['cookies']:
+            return self._config['cookies']
+        else:
+            return None
+
+    @cookies.setter
+    def cookies(self, cookies):
+        self._config['cookies'] = cookies
+
+    @property
+    def token(self):
+        if 'token' in self._config and self._config['token']:
+            return self._config['token']
+        else:
+            return None
+
+    @token.setter
+    def token(self, token):
+        self._config['token'] = token
+
+    @property
+    def max_attempts(self):
+        """Returns configured max_attempts or 1 if not configured"""
+        if 'max_attempts' in self._config and self._config['max_attempts']:
+            return self._config['max_attempts']
+        else:
+            return 1
+
+    @max_attempts.setter
+    def max_attempts(self, max_attempts):
+        self._config['max_attempts'] = max_attempts
+
+    @property
+    def min_retry_secs(self):
+        """Returns configured min_retry_seconds or 0 if not configured"""
+        if 'min_retry_secs' in self._config and self._config['min_retry_secs']:
+            return self._config['min_retry_secs']
+        else:
+            return 0
+
+    @min_retry_secs.setter
+    def min_retry_secs(self, min_retry_secs):
+        self._config['min_retry_secs'] = min_retry_secs
+
+    @property
+    def max_retry_secs(self):
+        """Returns configured max_retry_seconds or 0 if not configured"""
+        if 'max_retry_secs' in self._config and self._config['max_retry_secs']:
+            return self._config['max_retry_secs']
+        else:
+            return 0
+
+    @max_retry_secs.setter
+    def max_retry_secs(self, max_retry_secs):
+        self._config['max_retry_secs'] = max_retry_secs

pyxetabase/rest_clients/_parent_rest_clients.py

@@ -0,0 +1,110 @@
+import json
+
+from pyxetabase.commons import execute
+from pyxetabase.rest_response import RestResponse
+from pyxetabase.retry import retry
+
+
+class _ParentRestClient(object):
+    """Queries the REST service given the different query params"""
+
+    def __init__(self, configuration, token=None, login_handler=None,
+                 auto_refresh=True):
+        self.auto_refresh = auto_refresh
+        self._cfg = configuration
+        self.token = token
+        self.refresh_token = None
+        self.login_handler = login_handler
+        self.on_retry = None
+
+    def _client_login_handler(self):
+        if self.login_handler:
+            self.token, self.refresh_token = self.login_handler()
+
+    def _refresh_token_client(self):
+        if self.login_handler:
+            self.token, self.refresh_token = self.login_handler(refresh=True)
+
+    @staticmethod
+    def _get_query_id_str(query_ids):
+        if query_ids is None:
+            return None
+        elif isinstance(query_ids, list):
+            return ','.join(map(str, query_ids))
+        else:
+            return str(query_ids)
+
+    def _rest_retry(self, method, category, resource, query_id=None, subcategory=None,
+                    second_query_id=None, data=None, dont_retry=None,
+                    **options):
+
+        query_ids_str = self._get_query_id_str(query_id)
+
+        def exec_retry():
+            return execute(config=self._cfg,
+                           sid=self.token,
+                           category=category,
+                           subcategory=subcategory,
+                           method=method,
+                           query_id=query_ids_str,
+                           second_query_id=second_query_id,
+                           resource=resource,
+                           data=data,
+                           options=options)
+
+        def notify_retry(exc_type, exc_val, exc_tb):
+            if self.on_retry is not None:
+                self.on_retry(self, exc_type, exc_val, exc_tb, dict(
+                    method=method, resource=resource, query_id=query_id,
+                    category=category, subcategory=subcategory,
+                    second_query_id=second_query_id, data=data,
+                    options=options
+                ))
+
+        response = retry(
+            exec_retry, self._cfg.max_attempts, self._cfg.min_retry_secs,
+            self._cfg.max_retry_secs, login_handler=self.login_handler,
+            on_retry=notify_retry, dont_retry=dont_retry
+        )
+
+        if self.auto_refresh:
+            self._refresh_token_client()
+
+        if isinstance(response, dict):
+            return RestResponse(response)
+        else:  # Not a JSON (e.g. /{apiVersion}/files/{file}/download)
+            return response
+
+    def _get(self, category, resource, query_id=None, subcategory=None,
+             second_query_id=None, **options):
+        """Queries the REST service and returns the result"""
+        return self._rest_retry(
+            method='get', category=category, resource=resource, query_id=query_id,
+            subcategory=subcategory, second_query_id=second_query_id,
+            **options
+        )
+
+    def _post(self, category, resource, data=None, query_id=None, subcategory=None,
+              second_query_id=None, **options):
+        """Queries the REST service and returns the result"""
+        if data is not None:
+            return self._rest_retry(
+                method='post', category=category, resource=resource, query_id=query_id,
+                subcategory=subcategory, second_query_id=second_query_id,
+                data=data, **options
+            )
+        else:
+            return self._rest_retry(
+                method='post', category=category, resource=resource, query_id=query_id,
+                subcategory=subcategory, second_query_id=second_query_id,
+                **options
+            )
+
+    def _delete(self, category, resource, query_id=None, subcategory=None,
+                second_query_id=None, **options):
+        """Queries the REST service and returns the result"""
+        return self._rest_retry(
+            method='delete', category=category, resource=resource, query_id=query_id,
+            subcategory=subcategory, second_query_id=second_query_id,
+            **options
+        )