ebi-eva-common-pyutils 0.6.15__1-py3-none-any.whl

ebi_eva_common_pyutils/assembly/__init__.py

@@ -0,0 +1 @@
+from ebi_eva_common_pyutils.reference.assembly import NCBIAssembly

ebi_eva_common_pyutils/assembly/assembly.py

@@ -0,0 +1,69 @@
+# Copyright 2019 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from datetime import datetime
+from functools import lru_cache
+
+from ebi_eva_common_pyutils.logger import logging_config
+from ebi_eva_common_pyutils.network_utils import json_request
+from ebi_eva_common_pyutils.taxonomy.taxonomy import get_normalized_scientific_name_from_ensembl
+
+logger = logging_config.get_logger(__name__)
+
+
+def get_supported_asm_from_ensembl(tax_id: int) -> str:
+    logger.info(f'Query Ensembl for species name using taxonomy {tax_id}')
+    scientific_name_api_param = get_normalized_scientific_name_from_ensembl(tax_id)
+    ENSEMBL_REST_API_URL = "http://rest.ensembl.org/info/assembly/{0}?content-type=application/json".format(
+        scientific_name_api_param)
+    response = json_request(ENSEMBL_REST_API_URL)
+    assembly_accession_attribute = 'assembly_accession'
+    if assembly_accession_attribute in response:
+        return str(response.get(assembly_accession_attribute))
+    return None
+
+
+@lru_cache(maxsize=None)
+def get_taxonomy_to_assembly_mapping_from_ensembl_rapid_release():
+    """
+    Returns a dict mapping taxonomy ID to assembly accession, choosing the most recently released,
+    lexicographically last, non-alternate haplotype assembly when multiple are present.
+    """
+    list_data = json_request('https://ftp.ensembl.org/pub/rapid-release/species_metadata.json')
+    results = {}
+    for asm_data in list_data:
+        tax_id = asm_data['taxonomy_id']
+        asm_accession = asm_data['assembly_accession']
+        strain = asm_data['strain']
+        release_date = datetime.strptime(asm_data['release_date'], '%Y-%m-%d')
+
+        # If we haven't seen this taxonomy before, just use this assembly
+        if tax_id not in results:
+            results[tax_id] = (asm_accession, release_date)
+            continue
+
+        # Skip alternate haplotype assemblies
+        if strain and strain.lower() == 'alternate haplotype':
+            continue
+        current_assembly, current_date = results[tax_id]
+        # Keep the more recent assembly, or the lexicographically last one if release dates are equal
+        if current_date < release_date or (current_date == release_date and asm_accession > current_assembly):
+            results[tax_id] = (asm_accession, release_date)
+
+    return {key: val[0] for key, val in results.items()}
+
+
+def get_supported_asm_from_ensembl_rapid_release(tax_id: int) -> str:
+    # TODO: Replace with API call once supported
+    rapid_release_data = get_taxonomy_to_assembly_mapping_from_ensembl_rapid_release()
+    return rapid_release_data.get(tax_id, None)

ebi_eva_common_pyutils/assembly_utils.py

@@ -0,0 +1,91 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http
+import requests
+
+from ebi_eva_common_pyutils.assembly import NCBIAssembly
+from ebi_eva_common_pyutils.ena_utils import download_xml_from_ena
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+EUTILS_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
+ESEARCH_URL = EUTILS_URL + 'esearch.fcgi'
+ESUMMARY_URL = EUTILS_URL + 'esummary.fcgi'
+EFETCH_URL = EUTILS_URL + 'efetch.fcgi'
+
+
+logger = log_cfg.get_logger(__name__)
+
+
+def is_patch_assembly(assembly_accession: str) -> bool:
+    """
+    Check if a given assembly is a patch assembly
+    Please see: https://www.ncbi.nlm.nih.gov/grc/help/patches/
+    """
+    xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{assembly_accession}')
+    xml_assembly = xml_root.xpath("//ASSEMBLY_ATTRIBUTE[TAG='count-patches']/VALUE")
+    if len(xml_assembly) == 0:
+        return False
+    return int(xml_assembly[0].text) > 0
+
+
+def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt, api_key=None):
+    """
+    Attempt to find any assembly genebank accession base on a free text search.
+    """
+    assembly_accessions = set()
+    payload = {'db': 'Assembly', 'term': '"{}"'.format(assembly_txt), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
+    data = requests.get(ESEARCH_URL, params=payload).json()
+    if data and data.get('esearchresult', {}).get('idlist'):
+        assembly_id_list = data.get('esearchresult').get('idlist')
+        payload = {'db': 'Assembly', 'id': ','.join(assembly_id_list), 'retmode': 'JSON'}
+        if api_key:
+            payload['api_key'] = api_key
+        summary_list = requests.get(ESUMMARY_URL, params=payload).json()
+        for assembly_id in summary_list.get('result', {}).get('uids', []):
+            assembly_info = summary_list.get('result').get(assembly_id)
+            if 'genbank' in assembly_info['synonym']:
+                assembly_accessions.add(assembly_info['synonym']['genbank'])
+    if len(assembly_accessions) != 1:
+        logger.warning('%s Genbank synonyms found for assembly %s ', len(assembly_accessions), assembly_txt)
+    return list(assembly_accessions)
+
+
+def retrieve_genbank_equivalent_for_GCF_accession(assembly_accession, ncbi_api_key=None):
+    genbank_synonyms = retrieve_genbank_assembly_accessions_from_ncbi(assembly_accession, api_key=ncbi_api_key)
+    if len(genbank_synonyms) != 1:
+        raise ValueError('%s Genbank synonyms found for assembly %s ' % (len(genbank_synonyms), assembly_accession))
+    return genbank_synonyms.pop()
+
+
+def resolve_assembly_name_to_GCA_accession(assembly_name):
+    ENA_ASSEMBLY_NAME_QUERY_URL = "https://www.ebi.ac.uk/ena/portal/api/search" \
+                                  "?result=assembly&query=assembly_name%3D%22{0}%22&format=json".format(assembly_name)
+    response = requests.get(ENA_ASSEMBLY_NAME_QUERY_URL)
+    if response.status_code == http.HTTPStatus.OK.value:
+        response_json = response.json()
+        if len(response_json) == 0:
+            raise ValueError("Could not resolve assembly name {0} to a GCA accession!".format(assembly_name))
+        elif len(response_json) > 1:
+            raise ValueError("Assembly name {0} resolved to more than one GCA accession!".format(assembly_name))
+        else:
+            return response.json()[0]["accession"] + "." + response.json()[0]["version"]
+    else:
+        raise ValueError("Could not resolve assembly name {0} to a GCA accession!".format(assembly_name))
+
+
+def get_assembly_report_url(assembly_accession):
+    return NCBIAssembly(assembly_accession).assembly_report_url

ebi_eva_common_pyutils/biosamples_communicators.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import requests
+from functools import cached_property
+from ebi_eva_common_pyutils.logger import AppLogger
+from retry import retry
+
+
+class HALNotReadyError(Exception):
+    pass
+
+
+class HALCommunicator(AppLogger):
+    """
+    This class helps navigate through REST API that uses the HAL standard.
+    """
+    acceptable_code = [200, 201]
+
+    def __init__(self, auth_url, bsd_url, username, password):
+        self.auth_url = auth_url
+        self.bsd_url = bsd_url
+        self.username = username
+        self.password = password
+
+    def _validate_response(self, response):
+        """Check that the response has an acceptable code and raise if it does not"""
+        if response.status_code not in self.acceptable_code:
+            self.error(response.request.method + ': ' + response.request.url + " with " + str(response.request.body))
+            self.error("headers: {}".format(response.request.headers))
+            self.error("<{}>: {}".format(response.status_code, response.text))
+            raise ValueError('The HTTP status code ({}) is not one of the acceptable codes ({})'.format(
+                str(response.status_code), str(self.acceptable_code))
+            )
+        return response
+
+    @cached_property
+    def token(self):
+        """Retrieve the token from the AAP REST API then cache it for further quering"""
+        response = requests.get(self.auth_url, auth=(self.username, self.password))
+        self._validate_response(response)
+        return response.text
+
+    @retry(exceptions=(ValueError, requests.RequestException), tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+    def _req(self, method, url, **kwargs):
+        """Private method that sends a request using the specified method. It adds the headers required by bsd"""
+        headers = kwargs.pop('headers', {})
+        headers.update({'Accept': 'application/hal+json'})
+        if self.token is not None:
+            headers.update({'Authorization': 'Bearer ' + self.token})
+        if 'json' in kwargs:
+            headers['Content-Type'] = 'application/json'
+        response = requests.request(
+            method=method,
+            url=url,
+            headers=headers,
+            **kwargs
+        )
+        self._validate_response(response)
+        return response
+
+    def follows(self, query, json_obj=None, method='GET', url_template_values=None, join_url=None, **kwargs):
+        """
+        Finds a link within the json_obj using a query string or list, modify the link using the
+        url_template_values dictionary then query the link using the method and any additional keyword argument.
+        If the json_obj is not specified then it will use the root query defined by the base url.
+        """
+        all_pages = kwargs.pop('all_pages', False)
+
+        if json_obj is None:
+            json_obj = self.root
+        # Drill down into a dict using dot notation
+        _json_obj = json_obj
+        if isinstance(query, str):
+            query_list = query.split('.')
+        else:
+            query_list = query
+        for query_element in query_list:
+            if query_element in _json_obj:
+                _json_obj = _json_obj[query_element]
+            else:
+                raise KeyError('{} does not exist in json object'.format(query_element, _json_obj))
+        if not isinstance(_json_obj, str):
+            raise ValueError('The result of the query_string must be a string to use as a url')
+        url = _json_obj
+        # replace the template in the url with the value provided
+        if url_template_values:
+            for k, v in url_template_values.items():
+                url = re.sub('{(' + k + ')(:.*)?}', v, url)
+        if join_url:
+            url += '/' + join_url
+        text_only = False
+        if 'text_only' in kwargs and kwargs.get('text_only'):
+            text_only = kwargs.pop('text_only')
+        # Now query the url
+        response = self._req(method, url, **kwargs)
+        if text_only:
+            return response.text
+
+        json_response = response.json()
+        # Depaginate the call if requested
+        if all_pages is True:
+            # This depagination code will iterate over all the pages available until the pages comes back  without a
+            # next page. It stores the embedded elements in the initial query's json response
+            content = json_response
+            while 'next' in content.get('_links'):
+                content = self._req(method, content.get('_links').get('next').get('href'), **kwargs).json()
+                for key in content.get('_embedded'):
+                    json_response['_embedded'][key].extend(content.get('_embedded').get(key))
+            # Remove the pagination information as it is not relevant to the depaginated response
+            if 'page' in json_response: json_response.pop('page')
+            if 'first' in json_response['_links']: json_response['_links'].pop('first')
+            if 'last' in json_response['_links']: json_response['_links'].pop('last')
+            if 'next' in json_response['_links']: json_response['_links'].pop('next')
+        return json_response
+
+    def follows_link(self, key, json_obj=None, method='GET', url_template_values=None, join_url=None, **kwargs):
+        """
+        Same function as follows but construct the query_string from a single keyword surrounded by '_links' and 'href'.
+        """
+        return self.follows(('_links', key, 'href'),
+                            json_obj=json_obj, method=method, url_template_values=url_template_values,
+                            join_url=join_url, **kwargs)
+
+    @cached_property
+    def root(self):
+        return self._req('GET', self.bsd_url).json()
+
+    @property
+    def communicator_attributes(self):
+        raise NotImplementedError
+
+
+class AAPHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API using AAP authentication."""
+
+    def __init__(self, auth_url, bsd_url, username, password, domain=None):
+        super(AAPHALCommunicator, self).__init__(auth_url, bsd_url, username, password)
+        self.domain = domain
+
+    @property
+    def communicator_attributes(self):
+        return {'domain': self.domain}
+
+
+class WebinHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API using Webin authentication."""
+
+    @cached_property
+    def token(self):
+        """Retrieve the token from the ENA Webin REST API then cache it for further querying"""
+        response = requests.post(self.auth_url,
+                                 json={"authRealms": ["ENA"], "password": self.password,
+                                       "username": self.username})
+        self._validate_response(response)
+        return response.text
+
+    @property
+    def communicator_attributes(self):
+        return {'webinSubmissionAccountId': self.username}
+
+
+class NoAuthHALCommunicator(HALCommunicator):
+    """Class to navigate BioSamples API without authentication."""
+
+    def __init__(self, bsd_url):
+        super(NoAuthHALCommunicator, self).__init__(None, bsd_url, None, None)
+
+    @cached_property
+    def token(self):
+        """No auth token, so errors will be raised if auth is required for requests"""
+        return None

ebi_eva_common_pyutils/command_utils.py

@@ -0,0 +1,54 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import subprocess
+
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+logger = log_cfg.get_logger(__name__)
+
+
+def run_command_with_output(command_description, command, return_process_output=False,
+                            log_error_stream_to_output=False, stdout_log_level=logging.INFO,
+                            stderr_log_level=logging.ERROR):
+    process_output = ""
+
+    logger.log(stdout_log_level, "Starting process: " + command_description)
+    logger.log(stdout_log_level, "Running command: " + command)
+
+    stdout = subprocess.PIPE
+    # Some lame utilities like mongodump and mongorestore output non-error messages to error stream
+    # This is a workaround for that
+    stderr = subprocess.STDOUT if log_error_stream_to_output else subprocess.PIPE
+
+    with subprocess.Popen(command, stdout=stdout, stderr=stderr, bufsize=1, universal_newlines=True,
+                          shell=True) as process:
+        for line in iter(process.stdout.readline, ''):
+            line = str(line).rstrip()
+            logger.log(stdout_log_level, line)
+            if return_process_output:
+                process_output += line + "\n"
+        if not log_error_stream_to_output:
+            for line in iter(process.stderr.readline, ''):
+                line = str(line).rstrip()
+                logger.log(stderr_log_level, line)
+    if process.returncode != 0:
+        logger.error(command_description + " failed! Refer to the error messages for details.")
+        raise subprocess.CalledProcessError(process.returncode, process.args)
+    else:
+        logger.log(stdout_log_level, command_description + " - completed successfully")
+    if return_process_output:
+        return process_output
+
+

ebi_eva_common_pyutils/common_utils.py

@@ -0,0 +1,30 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def merge_two_dicts(x, y):
+    z = x.copy()  # start with x's keys and values
+    z.update(y)  # modifies z with y's keys and values & returns None
+    return z
+
+
+def pretty_print(header, table):
+    cell_widths = [len(h) for h in header]
+    for row in table:
+        for i, cell in enumerate(row):
+            cell_widths[i] = max(cell_widths[i], len(str(cell)))
+    format_string = ' | '.join('{%s:>%s}' % (i, w) for i, w in enumerate(cell_widths))
+    print('| ' + format_string.format(*header) + ' |')
+    for row in table:
+        print('| ' + format_string.format(*row) + ' |')

ebi_eva_common_pyutils/config.py

@@ -0,0 +1,152 @@
+import os
+
+import yaml
+
+
+class Configuration:
+    """
+    Configuration class that allow to load a yaml file either at construction or later in the execution.
+    It can be used like a dict but should be used as readonly.
+    """
+    config_file = None
+    content = {}
+
+    def __init__(self, *search_path):
+        if search_path:
+            self.load_config_file(*search_path)
+
+    @staticmethod
+    def _find_config_file(search_path):
+        for p in search_path:
+            if p and os.path.isfile(p):
+                return p
+
+    def load_config_file(self, *search_path):
+        self.config_file = self._find_config_file(search_path)
+        if self.config_file:
+            with open(self.config_file, 'r') as f:
+                self.content = yaml.safe_load(f)
+        else:
+            raise FileNotFoundError('Could not find any config file in specified search path')
+
+    def get(self, item, ret_default=None):
+        """
+        Dict-style item retrieval with default
+        :param item: The key to search for
+        :param ret_default: What to return if the key is not present
+        """
+        try:
+            return self[item]
+        except KeyError:
+            return ret_default
+
+    def query(self, *parts, ret_default=None):
+        """
+        Drill down into a config, e.g. cfg.query('logging', 'handlers', 'a_handler', 'level')
+        :param ret_default:
+        :return: The relevant item if it exists in the config, else ret_default.
+        """
+        top_level = self.content
+        item = None
+
+        for p in parts:
+            item = top_level.get(p)
+            if item:
+                top_level = item
+            else:
+                return ret_default
+        return item
+
+    def report(self):
+        return yaml.safe_dump(self.content, default_flow_style=False)
+
+    def __getitem__(self, item):
+        """Allow dict-style access, e.g. config['this'] or config['this']['that']."""
+        return self.content[item]
+
+    def __contains__(self, item):
+        """Allow search in the first layer of the config with 'in' operator."""
+        return self.content.__contains__(item)
+
+
+cfg = Configuration()
+"""
+Provides a singleton that can be used as a central place for configuration.
+"""
+
+
+
+
+class WritableConfig(Configuration):
+    """Configuration object that allows writes to the config file"""
+
+    def __init__(self, *search_path, version=None):
+        super().__init__(*search_path)
+        self.version = version
+
+    def load_config_file(self, *search_path):
+        try:
+            super().load_config_file(*search_path)
+        except FileNotFoundError:
+            # expected if it's the first time we are creating the config file
+            # In that case the first search path is set to be the config files
+            self.config_file = search_path[0]
+            pass
+
+    def backup(self):
+        """
+        Rename the config file by adding a '.1' at the end. If the '.1' file exists it move it to a '.2' and so on.
+        """
+        if os.path.isfile(self.config_file):
+            file_name = self.config_file
+            suffix = 1
+            backup_name = f'{file_name}.{suffix}'
+            while os.path.exists(backup_name):
+                suffix += 1
+                backup_name = f'{file_name}.{suffix}'
+
+            for i in range(suffix, 1, -1):
+                os.rename(f'{file_name}.{i - 1}', f'{file_name}.{i}')
+            os.rename(file_name, file_name + '.1')
+
+    def write(self):
+        if self.config_file and self.content and os.path.isdir(os.path.dirname(self.config_file)):
+            with open(self.config_file, 'w') as open_config:
+                yaml.safe_dump(self.content, open_config)
+
+    def set(self, *path, value):
+        self._set_version()
+        top_level = self.content
+        for p in path[:-1]:
+            if p not in top_level:
+                top_level[p] = {}
+            top_level = top_level[p]
+        top_level[path[-1]] = value
+
+    def pop(self, *path, default=None):
+        """Recursive dictionary pop with default"""
+        top_level = self.content
+        for p in path[:-1]:
+            if p not in top_level:
+                return default
+            top_level = top_level[p]
+        return top_level.pop(path[-1], default)
+
+    def is_empty(self):
+        return not self.content
+
+    def clear(self):
+        self.content = {}
+
+    def _set_version(self):
+        # If we're starting to fill in an empty config, set the version if available
+        if self.is_empty() and self.version:
+            self.content['version'] = self.version
+
+    def __contains__(self, item):
+        return item in self.content
+
+    def __setitem__(self, item, value):
+        """Allow dict-style write access, e.g. config['this']='that'."""
+        self._set_version()
+        self.content[item] = value