ebi-eva-common-pyutils 0.6.7__py3-none-any.whl → 0.6.9__py3-none-any.whl

ebi_eva_common_pyutils/biosamples_communicators.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import requests
+from functools import cached_property
+from ebi_eva_common_pyutils.logger import AppLogger
+from retry import retry
+
+
+class HALNotReadyError(Exception):
+    pass
+
+
+class HALCommunicator(AppLogger):
+    """
+    This class helps navigate through REST API that uses the HAL standard.
+    """
+    acceptable_code = [200, 201]
+
+    def __init__(self, auth_url, bsd_url, username, password):
+        self.auth_url = auth_url
+        self.bsd_url = bsd_url
+        self.username = username
+        self.password = password
+
+    def _validate_response(self, response):
+        """Check that the response has an acceptable code and raise if it does not"""
+        if response.status_code not in self.acceptable_code:
+            self.error(response.request.method + ': ' + response.request.url + " with " + str(response.request.body))
+            self.error("headers: {}".format(response.request.headers))
+            self.error("<{}>: {}".format(response.status_code, response.text))
+            raise ValueError('The HTTP status code ({}) is not one of the acceptable codes ({})'.format(
+                str(response.status_code), str(self.acceptable_code))
+            )
+        return response
+
+    @cached_property
+    def token(self):
+        """Retrieve the token from the AAP REST API then cache it for further quering"""
+        response = requests.get(self.auth_url, auth=(self.username, self.password))
+        self._validate_response(response)
+        return response.text
+
+    @retry(exceptions=(ValueError, requests.RequestException), tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+    def _req(self, method, url, **kwargs):
+        """Private method that sends a request using the specified method. It adds the headers required by bsd"""
+        headers = kwargs.pop('headers', {})
+        headers.update({'Accept': 'application/hal+json'})
+        if self.token is not None:
+            headers.update({'Authorization': 'Bearer ' + self.token})
+        if 'json' in kwargs:
+            headers['Content-Type'] = 'application/json'
+        response = requests.request(
+            method=method,
+            url=url,
+            headers=headers,
+            **kwargs
+        )
+        self._validate_response(response)
+        return response
+
+    def follows(self, query, json_obj=None, method='GET', url_template_values=None, join_url=None, **kwargs):
+        """
+        Finds a link within the json_obj using a query string or list, modify the link using the
+        url_template_values dictionary then query the link using the method and any additional keyword argument.
+        If the json_obj is not specified then it will use the root query defined by the base url.
+        """
+        all_pages = kwargs.pop('all_pages', False)
+
+        if json_obj is None:
+            json_obj = self.root
+        # Drill down into a dict using dot notation
+        _json_obj = json_obj
+        if isinstance(query, str):
+            query_list = query.split('.')
+        else:
+            query_list = query
+        for query_element in query_list:
+            if query_element in _json_obj:
+                _json_obj = _json_obj[query_element]
+            else:
+                raise KeyError('{} does not exist in json object'.format(query_element, _json_obj))
+        if not isinstance(_json_obj, str):
+            raise ValueError('The result of the query_string must be a string to use as a url')
+        url = _json_obj
+        # replace the template in the url with the value provided
+        if url_template_values:
+            for k, v in url_template_values.items():
+                url = re.sub('{(' + k + ')(:.*)?}', v, url)
+        if join_url:
+            url += '/' + join_url
+        # Now query the url
+        json_response = self._req(method, url, **kwargs).json()
+
+        # Depaginate the call if requested
+        if all_pages is True:
+            # This depagination code will iterate over all the pages available until the pages comes back  without a
+            # next page. It stores the embedded elements in the initial query's json response
+            content = json_response
+            while 'next' in content.get('_links'):
+                content = self._req(method, content.get('_links').get('next').get('href'), **kwargs).json()
+                for key in content.get('_embedded'):
+                    json_response['_embedded'][key].extend(content.get('_embedded').get(key))
+            # Remove the pagination information as it is not relevant to the depaginated response
+            if 'page' in json_response: json_response.pop('page')
+            if 'first' in json_response['_links']: json_response['_links'].pop('first')
+            if 'last' in json_response['_links']: json_response['_links'].pop('last')
+            if 'next' in json_response['_links']: json_response['_links'].pop('next')
+        return json_response
+
+    def follows_link(self, key, json_obj=None, method='GET', url_template_values=None, join_url=None, **kwargs):
+        """
+        Same function as follows but construct the query_string from a single keyword surrounded by '_links' and 'href'.
+        """
+        return self.follows(('_links', key, 'href'),
+                            json_obj=json_obj, method=method, url_template_values=url_template_values,
+                            join_url=join_url, **kwargs)
+
+    @cached_property
+    def root(self):
+        return self._req('GET', self.bsd_url).json()
+
+    @property
+    def communicator_attributes(self):
+        raise NotImplementedError
+
+
+class AAPHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API using AAP authentication."""
+
+    def __init__(self, auth_url, bsd_url, username, password, domain=None):
+        super(AAPHALCommunicator, self).__init__(auth_url, bsd_url, username, password)
+        self.domain = domain
+
+    @property
+    def communicator_attributes(self):
+        return {'domain': self.domain}
+
+
+class WebinHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API using Webin authentication."""
+
+    @cached_property
+    def token(self):
+        """Retrieve the token from the ENA Webin REST API then cache it for further querying"""
+        response = requests.post(self.auth_url,
+                                 json={"authRealms": ["ENA"], "password": self.password,
+                                       "username": self.username})
+        self._validate_response(response)
+        return response.text
+
+    @property
+    def communicator_attributes(self):
+        return {'webinSubmissionAccountId': self.username}
+
+
+class NoAuthHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API without authentication."""
+
+    def __init__(self, bsd_url):
+        super(NoAuthHALCommunicator, self).__init__(None, bsd_url, None, None)
+
+    @cached_property
+    def token(self):
+        """No auth token, so errors will be raised if auth is required for requests"""
+        return None

ebi_eva_common_pyutils/contig_alias/contig_alias.py

@@ -22,6 +22,9 @@ class InternalServerError(Exception):
     pass
 
 
+CONTING_ALIAS_URL = 'https://www.ebi.ac.uk/eva/webservices/contig-alias'
+
+
 # TODO add the get methods
 class ContigAliasClient(AppLogger):
     """
@@ -29,8 +32,13 @@ class ContigAliasClient(AppLogger):
     Authentication is required if using admin endpoints.
     """
 
-    def __init__(self, base_url, username=None, password=None):
-        self.base_url = base_url
+    def __init__(self, base_url=None, username=None, password=None, default_page_size=1000):
+        if base_url:
+            self.base_url = base_url
+        else:
+            self.base_url = os.environ.get('CONTING_ALIAS_URL') or CONTING_ALIAS_URL
+        # Used for get method
+        self.default_page_size=default_page_size
         # Only required for admin endpoints
         self.username = username
         self.password = password
@@ -69,3 +77,39 @@ class ContigAliasClient(AppLogger):
             raise InternalServerError
         else:
             self.error(f'Assembly accession {assembly} could not be deleted. Response: {response.text}')
+
+    @retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+    def _get_page_for_contig_alias_url(self, sub_url, page=0):
+        """queries the contig alias to retrieve the page of the provided url"""
+        url = f'{self.base_url}/{sub_url}?page={page}&size={self.default_page_size}'
+        response = requests.get(url, headers={'accept': 'application/json'})
+        response.raise_for_status()
+        response_json = response.json()
+        return response_json
+
+    def _depaginate_iter(self, sub_url, entity_to_retrieve):
+        """Generator that provides the contigs in the assembly requested."""
+        page = 0
+        response_json = self._get_page_for_contig_alias_url(sub_url, page=page)
+        for entity in response_json.get('_embedded', {}).get(entity_to_retrieve, []):
+            yield entity
+        while 'next' in response_json['_links']:
+            page += 1
+            response_json = self._get_page_for_contig_alias_url(sub_url, page=page)
+            for entity in response_json.get('_embedded', {}).get(entity_to_retrieve, []):
+                yield entity
+
+    def assembly_contig_iter(self, assembly_accession):
+        """Generator that provides the contigs in the assembly requested."""
+        sub_url = f'v1/assemblies/{assembly_accession}/chromosomes'
+        return self._depaginate_iter(sub_url, 'chromosomeEntities')
+
+    def assembly(self, assembly_accession):
+        """provides the description of the requested assembly."""
+        sub_url = f'v1/assemblies/{assembly_accession}'
+        response_json = self._get_page_for_contig_alias_url(sub_url)
+        return response_json.get('_embedded', {}).get('assemblyEntities', [])[0]
+
+    def contig_iter(self, insdc_accession):
+        sub_url = f'v1/chromosomes/genbank/{insdc_accession}'
+        return self._depaginate_iter(sub_url, 'chromosomeEntities')

{ebi_eva_common_pyutils-0.6.7.dist-info → ebi_eva_common_pyutils-0.6.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ebi-eva-common-pyutils
-Version: 0.6.7
+Version: 0.6.9
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache

{ebi_eva_common_pyutils-0.6.7.dist-info → ebi_eva_common_pyutils-0.6.9.dist-info}/RECORD

@@ -1,5 +1,6 @@
 ebi_eva_common_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_common_pyutils/assembly_utils.py,sha256=CklyCGlCjlFp0e9pugg6kSsh5L0xfCe2qPvA2eLVtn0,4187
+ebi_eva_common_pyutils/biosamples_communicators.py,sha256=ngWuSvEYkkttkMnJ9Gjxc5D7ZSi7D3_JX1c8BnEO3os,7596
 ebi_eva_common_pyutils/command_utils.py,sha256=PtelWWqcC0eOwIVesjwBw3F9KaXRzEE_uAUJhQFZ4l8,2340
 ebi_eva_common_pyutils/common_utils.py,sha256=ty_glvfRa3VGhnpAht4qtVkNNmv-IYfVtO958mY-BaA,1192
 ebi_eva_common_pyutils/config.py,sha256=PtD2SgHf96kk21OA9tVIjEgsDXEFuAU-INy_kfQdoPw,4828
@@ -11,7 +12,7 @@ ebi_eva_common_pyutils/network_utils.py,sha256=iJjs5PPzT1V4CceZnCHOTs711AmpwlDo5
 ebi_eva_common_pyutils/assembly/__init__.py,sha256=KSWPwBY5nZj00odxWFntk8Sqg_rw273xH8S5D6Jo-T4,67
 ebi_eva_common_pyutils/assembly/assembly.py,sha256=IEmleROX4ZchPyhINKCuMmET_Ih1Jg4ok-opAKY6Z9A,3142
 ebi_eva_common_pyutils/contig_alias/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ebi_eva_common_pyutils/contig_alias/contig_alias.py,sha256=8mxcIbOhivqedgDjpzsHEY3UbYwHvk9DGoEiyHLLOBk,3056
+ebi_eva_common_pyutils/contig_alias/contig_alias.py,sha256=7tQXljEJtRCMs_JPrglYvTEZ-fJvzl8txwnCvqRB3Ck,5214
 ebi_eva_common_pyutils/reference/__init__.py,sha256=NsDjGCu2H2CZLOPaEL6GCoZo5ND_EA9RFmbzGfpSXRQ,134
 ebi_eva_common_pyutils/reference/assembly.py,sha256=FK4zsvg4_GDeKd9z2z5ZeM9R84TsusgQYdMOGuggQU0,12162
 ebi_eva_common_pyutils/reference/sequence.py,sha256=bg96QcuB-oytQYpaUkV10OzM_RIFhxsy6t3PQOg2Gy0,3911
@@ -19,7 +20,7 @@ ebi_eva_common_pyutils/taxonomy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 ebi_eva_common_pyutils/taxonomy/taxonomy.py,sha256=p3XV4g3y0hEjyeZ4PwgN7Q3Et9G515ctQkSIo1kdDbU,2259
 ebi_eva_common_pyutils/variation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_common_pyutils/variation/contig_utils.py,sha256=kMNEW_P2yPnd8Xx1tep19hy5ee7ojxz6ZOO1grTQsRQ,5230
-ebi_eva_common_pyutils-0.6.7.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
+ebi_eva_common_pyutils-0.6.9.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
 ebi_eva_internal_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_internal_pyutils/archive_directory.py,sha256=IxVEfh_gaCiT652k0Q_-58fonRusy1yzXu7BCO8yVLo,4989
 ebi_eva_internal_pyutils/config_utils.py,sha256=EGRC5rsmU_ug7OY9-t1UW1XZXRsauSyZB9xPcBux8ts,7909
@@ -31,8 +32,8 @@ ebi_eva_internal_pyutils/mongodb/__init__.py,sha256=0oyTlkYZCV7udlPl09Zl-sDyE3c9
 ebi_eva_internal_pyutils/mongodb/mongo_database.py,sha256=kesaJaaxYFeF_uYZBgL8tbufGKUXll7bXb4WlOS9vKM,9596
 ebi_eva_internal_pyutils/nextflow/__init__.py,sha256=OOiJS8jZOz98q0t77NNog7aI_fFrVxi4kGmiSskuAqM,122
 ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py,sha256=ew623hhK8jmFLQjJwLZbgBmW9RTiJBEULVqHfIUv_dc,10114
-ebi_eva_common_pyutils-0.6.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ebi_eva_common_pyutils-0.6.7.dist-info/METADATA,sha256=CkS0gM_NHH4fr1TBxoXn6FTnZ4vfT4fS41ReZCKAy0U,824
-ebi_eva_common_pyutils-0.6.7.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-ebi_eva_common_pyutils-0.6.7.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
-ebi_eva_common_pyutils-0.6.7.dist-info/RECORD,,
+ebi_eva_common_pyutils-0.6.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ebi_eva_common_pyutils-0.6.9.dist-info/METADATA,sha256=iEw5N92Ad8rSkru7KNyzfpSgUD0NCpuF4Eu1dPSVp1I,824
+ebi_eva_common_pyutils-0.6.9.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+ebi_eva_common_pyutils-0.6.9.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
+ebi_eva_common_pyutils-0.6.9.dist-info/RECORD,,