msmetaenhancer 0.4.0__tar.gz

msmetaenhancer-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 RECETOX
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

msmetaenhancer-0.4.0/MSMetaEnhancer/__init__.py

@@ -0,0 +1,3 @@
+from MSMetaEnhancer.app import Application
+
+__all__ = ["Application"]

msmetaenhancer-0.4.0/MSMetaEnhancer/app.py

@@ -0,0 +1,103 @@
+import asyncio
+
+import aiohttp
+
+from MSMetaEnhancer.libs.Annotator import Annotator
+from MSMetaEnhancer.libs.Converter import Converter
+from MSMetaEnhancer.libs.Curator import Curator
+from MSMetaEnhancer.libs.data import Spectra, DataFrame
+from MSMetaEnhancer.libs.utils import logger
+from MSMetaEnhancer.libs.utils.ConverterBuilder import ConverterBuilder
+from MSMetaEnhancer.libs.utils.Errors import UnknownSpectraFormat
+from MSMetaEnhancer.libs.utils.Job import convert_to_jobs
+from MSMetaEnhancer.libs.utils.Monitor import Monitor
+
+
+class Application:
+    def __init__(self, log_level='info', log_file=None):
+        self.data = None
+        logger.setup(log_level, log_file)
+
+    def load_data(self, filename, file_format):
+        """
+        High level method to load Spectra data from given file.
+
+        :param filename: path to source spectra file
+        :param file_format: format of spectra
+        """
+        if file_format in ['msp', 'mgf', 'json']:
+            self.data = Spectra()
+        elif file_format in ['csv', 'tsv', 'xlsx']:
+            self.data = DataFrame()
+        else:
+            raise UnknownSpectraFormat(f'Format {file_format} not supported.')
+        self.data.load_data(filename, file_format)
+
+    def save_data(self, filename, file_format):
+        """
+        High level method to save Spectra data to given file.
+
+        :param filename: path to target file
+        :param file_format: desired format of spectra
+        """
+        self.data.save_data(filename, file_format)
+
+    def curate_metadata(self):
+        """
+        Updates metadata by curation process.
+
+        This includes e.g. normalisation of CAS numbers.
+        """
+        curated_metadata = Curator().curate_metadata(self.data.get_metadata())
+        self.data.fuse_metadata(curated_metadata)
+
+    async def annotate_spectra(self,
+                               converters,
+                               jobs=None,
+                               repeat: bool = False,
+                               monitor: Monitor = Monitor(),
+                               annotator: Annotator = Annotator()):
+        """
+        Annotates current Spectra data by specified jobs.
+
+        Used converters must be specified.
+        Jobs do not have to be given, all available jobs will be executed instead.
+
+        :param converters: given list of converters names
+        :param jobs: list specifying jobs to be executed
+        :param repeat: if some metadata was added, all jobs are executed again
+        :param monitor: given Monitor object to observe status of services
+        :param annotator: given Annotator object to run the actual annotation
+        """
+        async with aiohttp.ClientSession() as session:
+            builder = ConverterBuilder()
+            builder.validate_converters(converters)
+            converters, web_converters = builder.build_converters(session, converters)
+
+            annotator.set_converters(converters)
+            monitor.set_converters(web_converters)
+
+            # start converters status checker and wait for first status
+            try:
+                monitor.start()
+                monitor.first_check.wait()
+
+                # create all possible jobs if not given
+                if not jobs:
+                    jobs = []
+                    converter: Converter
+                    for converter in converters.values():
+                        jobs += converter.get_conversion_functions()
+                jobs = convert_to_jobs(jobs)
+
+                metadata_list = self.data.get_metadata()
+
+                logger.set_target_attributes(jobs, len(metadata_list))
+
+                results = await asyncio.gather(*[annotator.annotate(metadata, jobs, repeat)
+                                                 for metadata in metadata_list])
+            finally:
+                monitor.join()
+
+        self.data.fuse_metadata(results)
+        logger.write_metrics()

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/Annotator.py

@@ -0,0 +1,91 @@
+import traceback
+
+from MSMetaEnhancer.libs.Curator import Curator
+from MSMetaEnhancer.libs.utils import logger
+from MSMetaEnhancer.libs.utils.Errors import TargetAttributeNotRetrieved, SourceAttributeNotAvailable, \
+    ServiceNotAvailable, UnknownResponse, DataAlreadyPresent
+from MSMetaEnhancer.libs.utils.Logger import LogRecord
+
+
+class Annotator:
+    """
+    Annotator is responsible for annotation process of single spectra.
+    """
+    def __init__(self):
+        self.converters = dict()
+        self.curator = Curator()
+
+    def set_converters(self, converters):
+        self.converters = converters
+
+    async def annotate(self, metadata, jobs, repeat=False):
+        """
+        Runs all jobs to add annotations to given dictionary containing metadata
+
+        The method goes through specified jobs of form (Source, Target, Converter)
+        and tries to obtain 'Target' attribute based on 'Source' attribute using
+        'Converter' converter.
+
+        :param metadata: given metadata
+        :param jobs: specified list of jobs to be executed
+        :param repeat: if some metadata was added, all jobs are executed again
+        :return: annotated dictionary
+        """
+        cache = dict()
+        log = LogRecord(dict(metadata))
+        logger.add_coverage_before(metadata.keys())
+
+        added_metadata = True
+        while added_metadata:
+            added_metadata = False
+            for job in jobs:
+                if job.target not in metadata:
+                    try:
+                        metadata, cache = await self.execute_job_with_cache(job, metadata, cache, log)
+                        if repeat:
+                            added_metadata = True
+                    except (SourceAttributeNotAvailable, TargetAttributeNotRetrieved) as exc:
+                        log.update(exc, job, level=3)
+                    except (ServiceNotAvailable, UnknownResponse) as exc:
+                        log.update(exc, job, level=2)
+                    except Exception:
+                        log.update(Exception(traceback.format_exc()), job, level=1)
+                else:
+                    log.update(DataAlreadyPresent(f'Requested attribute {job.target} already present.'), job, level=2)
+
+        logger.add_logs(log)
+        logger.add_coverage_after(metadata.keys())
+
+        return metadata
+
+    async def execute_job_with_cache(self, job, metadata, cache, warning):
+        """
+        Execute given job in cached mode. Cache is converter specific
+        and spectra specific.
+
+        Raises TargetAttributeNotRetrieved
+
+        :param job: given job to be executed
+        :param metadata: data to be annotated by the job
+        :param cache: given cache for this spectra
+        :param warning: object storing warnings related to current metadata
+        :return: updated metadata and cache
+        """
+        # make sure the job makes sense
+        converter, data = job.validate(self.converters, metadata)
+
+        cache[job.converter] = cache.get(job.converter, dict())
+        if job.target in cache[job.converter]:
+            metadata[job.target] = cache[job.converter][job.target]
+        else:
+            if converter.is_available:
+                result = await converter.convert(job.source, job.target, data)
+                result = self.curator.filter_invalid_metadata(result, warning, job)
+                cache[job.converter].update(result)
+                if job.target in cache[job.converter]:
+                    metadata[job.target] = cache[job.converter][job.target]
+                else:
+                    raise TargetAttributeNotRetrieved('No data retrieved.')
+            else:
+                raise ServiceNotAvailable(f'Service {job.converter} not available.')
+        return metadata, cache

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/Converter.py

@@ -0,0 +1,81 @@
+from abc import ABC, abstractmethod
+
+
+class Converter(ABC):
+    """
+    General class for conversions.
+    """
+    def __init__(self):
+        self.is_available = True
+
+    @property
+    def converter_name(self):
+        return self.__class__.__name__
+
+    def __hash__(self):
+        return hash(self.converter_name)
+
+    @abstractmethod
+    async def convert(self, source, target, data):
+        """
+        Converts specified {source} attribute (provided in {data}) to {target} attribute.
+
+        :param source: given attribute name
+        :param target: required attribute name
+        :param data: given attribute value
+        :return: obtained value of target attribute
+        """
+        pass
+
+    def create_top_level_conversion_methods(self, conversions, asynch=True):
+        """
+        Method to create and set dynamic methods defined in conversions
+
+        :param conversions: triples of form (from, to, method)
+        :param asynch: whether to create asynchronous methods
+        """
+        for conversion in conversions:
+            create_top_level_method(self, *conversion, asynch)
+
+    def get_conversion_functions(self) -> list:
+        """
+        Method to compute all available conversion functions.
+
+        Assumes that the functions always have from {source}_to_{target}
+
+        :return: a list of available conversion functions
+        """
+        available_conversions = []
+        methods = [method_name for method_name in dir(self) if '_to_' in method_name]
+        for method in methods:
+            available_conversions.append((*method.split('_to_'), self.converter_name))
+        return available_conversions
+
+
+def create_top_level_method(obj: Converter, source: str, target: str, method: str, asynch: bool = True):
+    """
+    Assign a new method to {obj} called {source}_to_{target} which calls {method}.
+
+    :param obj: given object (typically a Converter)
+    :param source: attribute name used as source of data
+    :param target: attribute name obtained using this dynamic method
+    :param method: method which is called in the object with single argument
+    :param asynch: whether to create asynchronous methods
+    """
+    async def async_conversion(key):
+        return await getattr(obj, str(method))(key)
+
+    def sync_conversion(key):
+        return getattr(obj, str(method))(key)
+
+    doc = f'Convert {source} to {target} using {obj.__class__.__name__} converter'
+    name = f'{source}_to_{target}'
+
+    if asynch:
+        async_conversion.__doc__ = doc
+        async_conversion.__name__ = name
+        setattr(obj, async_conversion.__name__, async_conversion)
+    else:
+        sync_conversion.__doc__ = doc
+        sync_conversion.__name__ = name
+        setattr(obj, sync_conversion.__name__, sync_conversion)

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/Curator.py

@@ -0,0 +1,75 @@
+from matchms import metadata_utils
+from MSMetaEnhancer.libs.utils.Errors import InvalidAttributeFormat
+
+
+class Curator:
+    """
+    Curator makes sure that all data is curated before the actual annotation can proceed.
+    Currently, fixing CAS numbers to correct format is supported.
+
+    Additionally, it supports metadata validation to make sure the produced data are correct.
+    """
+    def curate_metadata(self, metadata_list):
+        """
+        Iterates over given metadata and curates individual entries.
+
+        :param metadata_list: given metadata
+        :return: curated metadata
+        """
+        curated_metadata_list = []
+        for metadata in metadata_list:
+            curated_metadata_list.append(self.curate_casno(metadata))
+        return curated_metadata_list
+
+    def curate_casno(self, metadata):
+        """
+        Curate metadata of particular spectra.
+
+        :param metadata: given metadata
+        :return: curated metadata
+        """
+        if 'casno' in metadata:
+            metadata['casno'] = self.fix_cas_number(metadata['casno'])
+        return metadata
+
+    @staticmethod
+    def fix_cas_number(cas_number):
+        """
+        Adds dashes to CAS number.
+
+        :param cas_number: given CAS number
+        :return: CAS number enriched by dashes (if needed)
+        """
+        if isinstance(cas_number, str):
+            if "-" not in cas_number:
+                return f'{cas_number[:-3]}-{cas_number[-3:-1]}-{cas_number[-1]}'
+        return cas_number
+
+    @staticmethod
+    def filter_invalid_metadata(metadata, log, job):
+        """
+        Validates metadata and filters out invalid ones.
+
+        :param metadata: metadata content
+        :param log: object storing logs related to current metadata
+        :param job: executed job
+        :return: only valid metadata
+        """
+        filters = {
+            'smiles': metadata_utils.is_valid_smiles,
+            'canonical_smiles': metadata_utils.is_valid_smiles,
+            'isomeric_smiles': metadata_utils.is_valid_smiles,
+            'inchi': metadata_utils.is_valid_inchi,
+            'inchikey': metadata_utils.is_valid_inchikey
+        }
+
+        valid_metadata = {}
+        for (attribute, value) in metadata.items():
+            if attribute in filters.keys():
+                if filters[attribute](value):
+                    valid_metadata[attribute] = value
+                else:
+                    log.update(InvalidAttributeFormat(f'Obtained {attribute} in invalid format: {value}'), job, level=2)
+            else:
+                valid_metadata[attribute] = value
+        return valid_metadata

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/converters/compute/ComputeConverter.py

@@ -0,0 +1,9 @@
+from MSMetaEnhancer.libs.Converter import Converter
+
+
+class ComputeConverter(Converter):
+    """
+    General class for computation conversion.
+    """
+    async def convert(self, source, target, data):
+        return getattr(self, f'{source}_to_{target}')(data)

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/converters/compute/RDKit.py

@@ -0,0 +1,70 @@
+import re
+from rdkit.Chem.Descriptors import ExactMolWt
+from rdkit.Chem import MolFromSmiles, MolToSmiles
+from rdkit.Chem.inchi import MolFromInchi
+from rdkit.Chem import Atom
+
+
+from MSMetaEnhancer.libs.converters.compute.ComputeConverter import ComputeConverter
+
+
+class RDKit(ComputeConverter):
+    """
+    RDKit is a collection of chemo-informatics and machine-learning software.
+    """
+    def __init__(self):
+        super().__init__()
+        # generate top level methods defining allowed conversions
+        conversions = [('smiles', 'mw', 'from_smiles'),
+                       ('canonical_smiles', 'mw', 'from_smiles'),
+                       ('isomeric_smiles', 'mw', 'from_smiles')]
+        self.create_top_level_conversion_methods(conversions, asynch=False)
+
+    def from_smiles(self, smiles):
+        """
+        Compute molecular exact weight from SMILES.
+
+        :param smiles: given SMILES
+        :return: computed molecular weight
+        """
+        weight = ExactMolWt(MolFromSmiles(smiles))
+        return {'mw': weight}
+
+    def inchi_to_canonical_smiles(self, inchi):
+        """
+        Compute canonical SMILES from InChI.
+
+        :param inchi: given InChI
+        :return: computed canonical SMILES
+        """
+        smiles = MolToSmiles(MolFromInchi(inchi), isomericSmiles=False)
+        return {'canonical_smiles': smiles}
+
+    def inchi_to_isomeric_smiles(self, inchi):
+        """
+        Compute isomeric SMILES from InChI.
+
+        :param inchi: given InChI
+        :return: computed isomeric SMILES
+        """
+        smiles = MolToSmiles(MolFromInchi(inchi))
+        return {'isomeric_smiles': smiles}
+
+    def formula_to_mw(self, formula):
+        """
+        Compute molecular exact weight from molecular formula.
+
+        :param formula: given formula
+        :return: computed molecular weight
+        """
+        parts = re.findall("[A-Z][a-z]?|[0-9]+", formula)
+        mass = 0
+
+        for index in range(len(parts)):
+            if parts[index].isnumeric():
+                continue
+
+            atom = Atom(parts[index])
+            multiplier = int(parts[index + 1]) if len(parts) > index + 1 and parts[index + 1].isnumeric() else 1
+            mass += atom.GetMass() * multiplier
+        return {'mw': mass}

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/converters/compute/__init__.py

@@ -0,0 +1,3 @@
+from MSMetaEnhancer.libs.converters.compute.RDKit import RDKit
+
+__all__ = ['RDKit']

msmetaenhancer-0.4.0/MSMetaEnhancer/libs/converters/web/BridgeDb.py

@@ -0,0 +1,135 @@
+from MSMetaEnhancer.libs.converters.web.WebConverter import WebConverter
+
+
+class BridgeDb(WebConverter):
+    """
+    BridgeDb is a framework to map identifiers between various biological databases. These mappings are provided for
+    genes, proteins, genetic variants, metabolites, and metabolic reactions
+
+    More info about the available conversions: https://bridgedb.github.io/
+    """
+    def __init__(self, session):
+        super().__init__(session)
+        # service URLs
+        self.endpoints = {'BridgeDb': 'https://webservice.bridgedb.org/Human/xrefs/'}
+
+        self.codes = {'hmdbid': 'Ch', 'pubchemid': 'Cpc', 'chemspiderid': 'Cs', 'wikidataid': 'Wd', 'chebiid': 'Ce',
+                      'keggid': 'Ck'}
+        self.identifiers = {'PubChem-compound': 'pubchemid', 'Chemspider': 'chemspiderid', 'ChEBI': 'chebiid',
+                            'HMDB': 'hmdbid', 'Wikidata': 'wikidataid', 'KEGG Compound': 'keggid'}
+
+        # generate top level methods defining allowed conversions
+        conversions = [('hmdbid', 'pubchemid', 'from_hmdbid'),
+                       ('hmdbid', 'chemspiderid', 'from_hmdbid'),
+                       ('hmdbid', 'wikidataid', 'from_hmdbid'),
+                       ('hmdbid', 'chebiid', 'from_hmdbid'),
+                       ('hmdbid', 'keggid', 'from_hmdbid'),
+                       ('pubchemid', 'hmdbid', 'from_pubchemid'),
+                       ('pubchemid', 'chemspiderid', 'from_pubchemid'),
+                       ('pubchemid', 'wikidataid', 'from_pubchemid'),
+                       ('pubchemid', 'chebiid', 'from_pubchemid'),
+                       ('pubchemid', 'keggid', 'from_pubchemid'),
+                       ('chemspiderid', 'hmdbid', 'from_chemspiderid'),
+                       ('chemspiderid', 'pubchemid', 'from_chemspiderid'),
+                       ('chemspiderid', 'wikidataid', 'from_chemspiderid'),
+                       ('chemspiderid', 'chebiid', 'from_chemspiderid'),
+                       ('chemspiderid', 'keggid', 'from_chemspiderid'),
+                       ('wikidataid', 'hmdbid', 'from_wikidataid'),
+                       ('wikidataid', 'pubchemid', 'from_wikidataid'),
+                       ('wikidataid', 'chemspiderid', 'from_wikidataid'),
+                       ('wikidataid', 'chebiid', 'from_wikidataid'),
+                       ('wikidataid', 'keggid', 'from_wikidataid'),
+                       ('chebiid', 'hmdbid', 'from_chebiid'),
+                       ('chebiid', 'pubchemid', 'from_chebiid'),
+                       ('chebiid', 'chemspiderid', 'from_chebiid'),
+                       ('chebiid', 'wikidataid', 'from_chebiid'),
+                       ('chebiid', 'keggid', 'from_chebiid'),
+                       ('keggid', 'hmdbid', 'from_keggid'),
+                       ('keggid', 'pubchemid', 'from_keggid'),
+                       ('keggid', 'chemspiderid', 'from_keggid'),
+                       ('keggid', 'wikidataid', 'from_keggid'),
+                       ('keggid', 'chebiid', 'from_keggid'),
+                       ]
+        self.create_top_level_conversion_methods(conversions)
+
+    async def from_hmdbid(self, hmdbid):
+        """
+        Convert HMDB ID to all possible IDs using BridgeDb web service
+
+        :param hmdbid: given HMDB ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["hmdbid"]}/{hmdbid}'
+        return await self.call_service(args)
+
+    async def from_pubchemid(self, pubchemid):
+        """
+        Convert PubChem ID to all possible IDs using BridgeDb web service
+
+        :param pubchemid: given PubChem ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["pubchemid"]}/{pubchemid}'
+        return await self.call_service(args)
+
+    async def from_chemspiderid(self, chemspiderid):
+        """
+        Convert ChemSpider ID to all possible IDs using BridgeDb web service
+
+        :param chemspiderid: given ChemSpider ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["chemspiderid"]}/{chemspiderid}'
+        return await self.call_service(args)
+
+    async def from_wikidataid(self, wikidataid):
+        """
+        Convert WikiData ID to all possible IDs using BridgeDb web service
+
+        :param wikidataid: given WikiData ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["wikidataid"]}/{wikidataid}'
+        return await self.call_service(args)
+
+    async def from_chebiid(self, chebiid):
+        """
+        Convert ChEBI ID to all possible IDs using BridgeDb web service
+
+        :param chebiid: given ChEBI ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["chebiid"]}/{chebiid}'
+        return await self.call_service(args)
+
+    async def from_keggid(self, keggid):
+        """
+        Convert KEGG ID to all possible IDs using BridgeDb web service
+
+        :param keggid: given KEGG ID number
+        :return: obtained IDs
+        """
+        args = f'{self.codes["keggid"]}/{keggid}'
+        return await self.call_service(args)
+
+    async def call_service(self, args):
+        response = await self.query_the_service('BridgeDb', args)
+        if response:
+            return self.parse_attributes(response)
+
+    def parse_attributes(self, response):
+        """
+        Parse all available attributes obtained using BridgeDb.
+
+        :param response: BridgeDb response to given ID
+        :return: all parsed data
+        """
+        result = dict()
+
+        lines = response.split('\n')
+        for line in lines:
+            if line:
+                value, identifier = line.split('\t')
+                if identifier in self.identifiers.keys():
+                    result[self.identifiers[identifier]] = value
+        return result