ebi-eva-common-pyutils 0.6.15__1-py3-none-any.whl

ebi_eva_common_pyutils/contig_alias/contig_alias.py

@@ -0,0 +1,115 @@
+# Copyright 2022 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from ebi_eva_common_pyutils.logger import AppLogger
+import requests
+from retry import retry
+
+
+class InternalServerError(Exception):
+    pass
+
+
+CONTING_ALIAS_URL = 'https://www.ebi.ac.uk/eva/webservices/contig-alias'
+
+
+# TODO add the get methods
+class ContigAliasClient(AppLogger):
+    """
+    Python client for interfacing with the contig alias service.
+    Authentication is required if using admin endpoints.
+    """
+
+    def __init__(self, base_url=None, username=None, password=None, default_page_size=1000):
+        if base_url:
+            self.base_url = base_url
+        else:
+            self.base_url = os.environ.get('CONTING_ALIAS_URL') or CONTING_ALIAS_URL
+        # Used for get method
+        self.default_page_size=default_page_size
+        # Only required for admin endpoints
+        self.username = username
+        self.password = password
+
+    def check_auth(self):
+        if self.username is None or self.password is None:
+            raise ValueError('Need admin username and password for this method')
+
+    @retry(InternalServerError, tries=3, delay=2, backoff=1.5, jitter=(1, 3))
+    def insert_assembly(self, assembly):
+        self.check_auth()
+        full_url = os.path.join(self.base_url, f'v1/admin/assemblies/{assembly}')
+
+        response = requests.put(full_url, auth=(self.username, self.password))
+        if response.status_code == 200:
+            self.info(f'Assembly accession {assembly} successfully added to Contig-Alias DB')
+        elif response.status_code == 409:
+            self.warning(f'Assembly accession {assembly} already exists in Contig-Alias DB. Response: {response.text}')
+        elif response.status_code == 500:
+            self.error(f'Could not save Assembly accession {assembly} to Contig-Alias DB. Error: {response.text}')
+            raise InternalServerError
+        else:
+            self.error(f'Could not save Assembly accession {assembly} to Contig-Alias DB. Error: {response.text}')
+            response.raise_for_status()
+
+    @retry(InternalServerError, tries=3, delay=2, backoff=1.5, jitter=(1, 3))
+    def delete_assembly(self, assembly):
+        self.check_auth()
+        full_url = os.path.join(self.base_url, f'v1/admin/assemblies/{assembly}')
+
+        response = requests.delete(full_url, auth=(self.username, self.password))
+        if response.status_code == 200:
+            self.info(f'Assembly accession {assembly} successfully deleted from Contig-Alias DB')
+        elif response.status_code == 500:
+            self.error(f'Assembly accession {assembly} could not be deleted. Response: {response.text}')
+            raise InternalServerError
+        else:
+            self.error(f'Assembly accession {assembly} could not be deleted. Response: {response.text}')
+
+    @retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+    def _get_page_for_contig_alias_url(self, sub_url, page=0):
+        """queries the contig alias to retrieve the page of the provided url"""
+        url = f'{self.base_url}/{sub_url}?page={page}&size={self.default_page_size}'
+        response = requests.get(url, headers={'accept': 'application/json'})
+        response.raise_for_status()
+        response_json = response.json()
+        return response_json
+
+    def _depaginate_iter(self, sub_url, entity_to_retrieve):
+        """Generator that provides the contigs in the assembly requested."""
+        page = 0
+        response_json = self._get_page_for_contig_alias_url(sub_url, page=page)
+        for entity in response_json.get('_embedded', {}).get(entity_to_retrieve, []):
+            yield entity
+        while 'next' in response_json['_links']:
+            page += 1
+            response_json = self._get_page_for_contig_alias_url(sub_url, page=page)
+            for entity in response_json.get('_embedded', {}).get(entity_to_retrieve, []):
+                yield entity
+
+    def assembly_contig_iter(self, assembly_accession):
+        """Generator that provides the contigs in the assembly requested."""
+        sub_url = f'v1/assemblies/{assembly_accession}/chromosomes'
+        return self._depaginate_iter(sub_url, 'chromosomeEntities')
+
+    def assembly(self, assembly_accession):
+        """provides the description of the requested assembly."""
+        sub_url = f'v1/assemblies/{assembly_accession}'
+        response_json = self._get_page_for_contig_alias_url(sub_url)
+        return response_json.get('_embedded', {}).get('assemblyEntities', [])[0]
+
+    def contig_iter(self, insdc_accession):
+        sub_url = f'v1/chromosomes/genbank/{insdc_accession}'
+        return self._depaginate_iter(sub_url, 'chromosomeEntities')

ebi_eva_common_pyutils/ena_utils.py

@@ -0,0 +1,35 @@
+import requests
+from lxml import etree
+from retry import retry
+
+
+@retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def download_xml_from_ena(ena_url) -> etree.XML:
+    """Download and parse XML from ENA"""
+    try:  # catches any kind of request error, including non-20X status code
+        response = requests.get(ena_url)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as e:
+        raise e
+    root = etree.XML(bytes(response.text, encoding='utf-8'))
+    return root
+
+
+def get_assembly_name_and_taxonomy_id(assembly_accession):
+    xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{assembly_accession}')
+    xml_assembly = xml_root.xpath('/ASSEMBLY_SET/ASSEMBLY')
+    if len(xml_assembly) == 0:
+        raise ValueError(f'Assembly {assembly_accession} not found in ENA')
+    assembly_name = xml_assembly[0].get('alias')
+    taxonomy_id = int(xml_assembly[0].xpath('TAXON/TAXON_ID')[0].text)
+    return assembly_name, taxonomy_id
+
+
+def get_scientific_name_and_common_name(taxonomy_id):
+    xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{taxonomy_id}')
+    xml_taxon = xml_root.xpath('/TAXON_SET/taxon')
+    if len(xml_taxon) == 0:
+        raise ValueError(f'Taxonomy {taxonomy_id} not found in ENA')
+    scientific_name = xml_taxon[0].get('scientificName')
+    optional_common_name = xml_taxon[0].get('commonName')
+    return scientific_name, optional_common_name

ebi_eva_common_pyutils/file_utils.py

@@ -0,0 +1,31 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from enum import Enum
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+
+
+class FileDiffOption(Enum):
+    NOT_IN = 1
+    COMMON = 2
+
+
+def file_diff(file1_path: str, file2_path: str, diff_option: FileDiffOption, output_file_path: str):
+    if diff_option == FileDiffOption.NOT_IN:
+        run_command_with_output("Finding entries in {0} not in {1}".format(file1_path, file2_path),
+                                "comm -23 {0} {1} > {2}".format(file1_path, file2_path, output_file_path))
+    elif diff_option == FileDiffOption.COMMON:
+        run_command_with_output("Finding entries common to {0} and {1}".format(file1_path, file2_path),
+                                "comm -12 {0} {1} > {2}".format(file1_path, file2_path, output_file_path))

ebi_eva_common_pyutils/logger.py

@@ -0,0 +1,150 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import logging.config
+import logging.handlers
+from sys import stdout, stderr
+from cached_property import cached_property
+
+
+class LoggingConfiguration:
+    """
+    This class provides an all in one management of all loggers in the stack. By default it pulls existing loggers,
+    stores additional ones along with handlers and formatters.
+    """
+
+    default_fmt = '[%(asctime)s][%(name)s][%(levelname)s] %(message)s'
+    default_datefmt = '%Y-%b-%d %H:%M:%S'
+
+    def __init__(self, use_existing_logger=True, log_level=logging.INFO):
+        self.blank_formatter = logging.Formatter()
+        self.handlers = set()
+        if use_existing_logger:
+            # retrieve all third party loggers
+            self.loggers = dict((name, logger)
+                                for name, logger in logging.root.manager.loggerDict.items()
+                                if isinstance(logger, logging.Logger))
+        else:
+            self.loggers = {}
+        self._log_level = log_level
+
+    @cached_property
+    def formatter(self):
+        return self.default_formatter
+
+    @cached_property
+    def default_formatter(self):
+        return logging.Formatter(
+            fmt=self.default_fmt,
+            datefmt=self.default_datefmt
+        )
+
+    def get_logger(self, name, level=logging.NOTSET):
+        """
+        Return a logging.Logger object with formatters and handlers added.
+        :param name: Name to assign to the logger (usually __name__)
+        :param int level: Log level to assign to the logger upon creation
+        """
+        if name in self.loggers:
+            logger = self.loggers[name]
+        else:
+            logger = logging.getLogger(name)
+            self.loggers[name] = logger
+
+        logger.setLevel(level or self._log_level)
+        for h in self.handlers:
+            logger.addHandler(h)
+
+        return logger
+
+    def add_handler(self, handler, level=logging.NOTSET):
+        """
+        Add a created handler, set its format/level if needed and register all loggers to it
+        :param logging.Handler handler:
+        :param int level: Log level to assign to the created handler
+        """
+        handler.setLevel(level or self._log_level)
+        handler.setFormatter(self.formatter)
+        for name in self.loggers:
+            self.loggers[name].addHandler(handler)
+        self.handlers.add(handler)
+
+    def add_stdout_handler(self, level=None):
+        self.add_handler(logging.StreamHandler(stdout), level=level or self._log_level)
+
+    def add_stderr_handler(self, level=None):
+        self.add_handler(logging.StreamHandler(stderr), level=level or self._log_level)
+
+    def add_file_handler(self, filename, level=None):
+        self.add_handler(logging.FileHandler(filename=filename), level=level or self._log_level)
+
+    def set_log_level(self, level):
+        self._log_level = level
+        for h in self.handlers:
+            h.setLevel(self._log_level)
+        for name in self.loggers:
+            self.loggers[name].setLevel(self._log_level)
+
+    def set_formatter(self, formatter):
+        """
+        Set all handlers to use formatter
+        :param logging.Formatter formatter:
+        """
+        self.__dict__['formatter'] = formatter
+        for h in self.handlers:
+            h.setFormatter(self.formatter)
+
+    def reset(self):
+        """Remove all handlers of existing logger"""
+        for l in self.loggers.values():
+            while l.handlers:
+                l.removeHandler(l.handlers[0])
+
+        while self.handlers:
+            h = self.handlers.pop()
+            del h
+
+
+# A logging configuration singleton that will be the only source of logger
+logging_config = LoggingConfiguration()
+
+
+class AppLogger:
+    """
+    Mixin class for logging. An object subclassing this can log using its class name. Contains a
+    logging.Logger object and exposes its log methods.
+    """
+    log_cfg = logging_config
+
+    def log(self, level, msg, *args, **kwargs):
+        self._logger.log(level, msg, *args, **kwargs)
+
+    def debug(self, msg, *args):
+        self._logger.debug(msg, *args)
+
+    def info(self, msg, *args):
+        self._logger.info(msg, *args)
+
+    def warning(self, msg, *args):
+        self._logger.warning(msg, *args)
+
+    def error(self, msg, *args):
+        self._logger.error(msg, *args)
+
+    def critical(self, msg, *args):
+        self._logger.critical(msg, *args)
+
+    @cached_property
+    def _logger(self):
+        return self.log_cfg.get_logger(self.__class__.__name__)

ebi_eva_common_pyutils/ncbi_utils.py

@@ -0,0 +1,117 @@
+import re
+
+import requests
+from retry import retry
+
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+
+logger = log_cfg.get_logger(__name__)
+
+eutils_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
+esearch_url = eutils_url + 'esearch.fcgi'
+esummary_url = eutils_url + 'esummary.fcgi'
+efetch_url = eutils_url + 'efetch.fcgi'
+ensembl_url = 'http://rest.ensembl.org/info/assembly'
+
+
+@retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def get_ncbi_assembly_dicts_from_term(term, api_key=None):
+    """Function to return NCBI assembly objects in the form of a list of dictionaries based on a search term."""
+    payload = {'db': 'Assembly', 'term': '"{}"'.format(term), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
+    req = requests.get(esearch_url, params=payload)
+    req.raise_for_status()
+    data = req.json()
+    assembly_dicts = []
+    if data:
+        assembly_id_list = data.get('esearchresult').get('idlist')
+        payload = {'db': 'Assembly', 'id': ','.join(assembly_id_list), 'retmode': 'JSON'}
+        if api_key:
+            payload['api_key'] = api_key
+        req = requests.get(esummary_url, params=payload)
+        req.raise_for_status()
+        summary_list = req.json()
+        for assembly_id in summary_list.get('result', {}).get('uids', []):
+            assembly_dicts.append(summary_list.get('result').get(assembly_id))
+    return assembly_dicts
+
+
+@retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def get_ncbi_taxonomy_dicts_from_term(term, api_key=None):
+    """Function to return NCBI taxonomy objects in the form of a list of dictionaries based on a search term."""
+    payload = {'db': 'Taxonomy', 'term': '"{}"'.format(term), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
+    req = requests.get(esearch_url, params=payload)
+    req.raise_for_status()
+    data = req.json()
+    taxonomy_dicts = []
+    if data:
+        taxonomy_dicts = get_ncbi_taxonomy_dicts_from_ids(data.get('esearchresult').get('idlist'))
+    return taxonomy_dicts
+
+
+@retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def get_ncbi_taxonomy_dicts_from_ids(taxonomy_ids, api_key=None):
+    """Function to return NCBI taxonomy objects in the form of a list of dictionaries
+    based on a list of taxonomy ids."""
+    taxonomy_dicts = []
+    payload = {'db': 'Taxonomy', 'id': ','.join(taxonomy_ids), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
+    req = requests.get(esummary_url, params=payload)
+    req.raise_for_status()
+    summary_list = req.json()
+    for taxonomy_id in summary_list.get('result', {}).get('uids', []):
+        taxonomy_dicts.append(summary_list.get('result').get(taxonomy_id))
+    return taxonomy_dicts
+
+
+def get_ncbi_assembly_name_from_term(term, api_key=None):
+    assembl_dicts = get_ncbi_assembly_dicts_from_term(term, api_key=api_key)
+    assembly_names = set([d.get('assemblyname') for d in assembl_dicts])
+    if len(assembly_names) > 1:
+        # Only keep the one that have the assembly accession as a synonymous and check again
+        assembly_names = set([d.get('assemblyname') for d in assembl_dicts
+                              if term in d['synonym'].values() or term == d['assemblyaccession']])
+    if len(assembly_names) != 1:
+        raise ValueError(f'Cannot resolve assembly name for assembly {term} in NCBI. '
+                         f'Found {",".join([str(a) for a in assembly_names])}')
+    return assembly_names.pop() if assembly_names else None
+
+
+def retrieve_species_scientific_name_from_tax_id_ncbi(taxid, api_key=None):
+    payload = {'db': 'Taxonomy', 'id': taxid}
+    if api_key:
+        payload['api_key'] = api_key
+    r = requests.get(efetch_url, params=payload)
+    match = re.search('<Rank>(.+?)</Rank>', r.text, re.MULTILINE)
+    rank = None
+    if match:
+        rank = match.group(1)
+    if rank not in ['species', 'subspecies']:
+        logger.warning('Taxonomy id %s does not point to a species', taxid)
+    match = re.search('<ScientificName>(.+?)</ScientificName>', r.text, re.MULTILINE)
+    if match:
+        return match.group(1)
+
+
+def get_species_name_from_ncbi(assembly_acc, api_key=None):
+    # We first need to search for the species associated with the assembly
+    assembly_dicts = get_ncbi_assembly_dicts_from_term(assembly_acc, api_key=api_key)
+    taxids = set([assembly_dict.get('taxid')
+        for assembly_dict in assembly_dicts
+        if assembly_dict.get('assemblyaccession') == assembly_acc or
+           assembly_dict.get('synonym', {}).get('genbank') == assembly_acc])
+
+    # This is a search so could retrieve multiple results
+    if len(taxids) != 1:
+        raise ValueError(f'Multiple species found for {assembly_acc}. '
+                         f'Cannot resolve single species for assembly {assembly_acc} in NCBI.')
+
+    taxonomy_id = taxids.pop()
+
+    scientific_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id, api_key=api_key)
+    return scientific_name.replace(' ', '_').lower()

ebi_eva_common_pyutils/network_utils.py

@@ -0,0 +1,64 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+
+import requests
+import subprocess
+from retry import retry
+
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+logger = log_cfg.get_logger(__name__)
+
+
+def is_port_in_use(port):
+    import socket
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(('localhost', port)) == 0
+
+
+def get_available_local_port(try_starting_with_port):
+    for i in range(0, 20):
+        port_to_try = try_starting_with_port + i
+        logger.info("Attempting to forward remote mongo port to local port {0}...".format(port_to_try))
+        if is_port_in_use(port_to_try):
+            logger.info("Port {0} already in use...".format(port_to_try))
+        else:
+            return port_to_try
+    logger.error("Could not forward to any local port!")
+
+
+def forward_remote_port_to_local_port(remote_host: str, remote_port: int, local_port: int) -> int:
+    port_forward_command = 'ssh -N -L{0}:localhost:{1} {2}'.format(local_port, remote_port, remote_host)
+    logger.info("Forwarding port to local port using command: " + port_forward_command)
+    proc = subprocess.Popen(port_forward_command.split(" "))
+    time.sleep(5)
+    # Ensure that the process is still running
+    poll = proc.poll()
+    if poll is not None:
+        # The process already completed which mean it most likely crashed
+        logger.error(f'Port Forwarding {remote_host}:{remote_port} -> {local_port} failed!')
+        raise subprocess.CalledProcessError(proc.returncode, proc.args)
+    return proc.pid
+
+
+@retry(exceptions=(ConnectionError, requests.RequestException), logger=logger,
+       tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+def json_request(url: str, payload: dict = None, method=requests.get) -> dict:
+    """Makes a request of a specified type (by default GET) with the specified URL and payload, attempts to parse the
+    result as a JSON string and return it as a dictionary, on failure raises an exception."""
+    result = method(url, data=payload)
+    result.raise_for_status()
+    return result.json()
+

ebi_eva_common_pyutils/reference/__init__.py

@@ -0,0 +1,2 @@
+from ebi_eva_common_pyutils.reference.assembly import NCBIAssembly
+from ebi_eva_common_pyutils.reference.sequence import NCBISequence