biotite 1.1.0__cp313-cp313-win_amd64.whl

biotite/database/entrez/__init__.py

@@ -0,0 +1,15 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+"""
+A subpackage for downloading files from the NCBI Entrez database.
+"""
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann"
+
+from .dbnames import *
+from .download import *
+from .key import *
+from .query import *

biotite/database/entrez/check.py

@@ -0,0 +1,60 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann, Maximilian Dombrowsky"
+__all__ = ["check_for_errors"]
+
+import json
+from biotite.database.error import RequestError
+
+# Taken from https://github.com/kblin/ncbi-entrez-error-messages
+_error_messages = [
+    "Error reading from remote server",
+    "Bad gateway",
+    "Bad Gateway",
+    "Cannot process ID list",
+    "server is temporarily unable to service your request",
+    "Service unavailable",
+    "Server Error",
+    "ID list is empty",
+    "Supplied id parameter is empty",
+    "Resource temporarily unavailable",
+    "Failed to retrieve sequence",
+    "Failed to understand id",
+]
+
+
+def check_for_errors(message):
+    """
+    Check for common error messages in NCBI Entrez database responses.
+
+    Parameters
+    ----------
+    message : str
+        The message received from NCBI Entrez.
+
+    Raises
+    ------
+    RequestError
+        If the message contains an error message.
+    """
+    # Server can respond short JSON error messages
+    if len(message) < 500:
+        try:
+            message_json = json.loads(message)
+            if "error" in message_json:
+                raise RequestError(message_json["error"])
+        except json.decoder.JSONDecodeError:
+            # It is not a JSON message
+            pass
+
+    # Error always appear at the end of message
+    message_end = message[-200:]
+    # Seemingly arbitrary '+' characters are in NCBI error messages
+    message_end = message_end.replace("+", "")
+    for error_msg in _error_messages:
+        # Often whitespace is also replaced by '+' in error message
+        if error_msg.replace(" ", "") in message_end:
+            raise RequestError(error_msg)

biotite/database/entrez/dbnames.py

@@ -0,0 +1,91 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann"
+__all__ = ["get_database_name"]
+
+
+# fmt: off
+_db_names = {
+    "BioProject"        : "bioproject",
+    "BioSample"         : "biosample",
+    "Biosystems"        : "biosystems",
+    "Books"             : "books",
+    "Conserved Domains" : "cdd",
+    "dbGaP"             : "gap",
+    "dbVar"             : "dbvar",
+    "Epigenomics"       : "epigenomics",
+    "EST"               : "nucest",
+    "Gene"              : "gene",
+    "Genome"            : "genome",
+    "GEO Datasets"      : "gds",
+    "GEO Profiles"      : "geoprofiles",
+    "GSS"               : "nucgss",
+    "HomoloGene"        : "homologene",
+    "MeSH"              : "mesh",
+    "NCBI C++ Toolkit"  : "toolkit",
+    "NCBI Web Site"     : "ncbisearch",
+    "NLM Catalog"       : "nlmcatalog",
+    "Nucleotide"        : "nuccore",
+    "OMIA"              : "omia",
+    "PopSet"            : "popset",
+    "Probe"             : "probe",
+    "Protein"           : "protein",
+    "Protein Clusters"  : "proteinclusters",
+    "PubChem BioAssay"  : "pcassay",
+    "PubChem Compound"  : "pccompound",
+    "PubChem Substance" : "pcsubstance",
+    "PubMed"            : "pubmed",
+    "PubMed Central"    : "pmc",
+    "SNP"               : "snp",
+    "SRA"               : "sra",
+    "Structure"         : "structure",
+    "Taxonomy"          : "taxonomy",
+    "UniGene"           : "unigene",
+    "UniSTS"            : "unists"
+}
+# fmt: on
+
+
+def get_database_name(database):
+    """
+    Map a common NCBI Entrez database name to an E-utility database
+    name.
+
+    Parameters
+    ----------
+    database : str
+        Entrez database name.
+
+    Returns
+    -------
+    name : str
+        E-utility database name.
+
+    Examples
+    --------
+
+    >>> print(get_database_name("Nucleotide"))
+    nuccore
+    """
+    return _db_names[database]
+
+
+def sanitize_database_name(db_name):
+    """
+    Map a common NCBI Entrez database name to an E-utility database
+    name, return E-utility database name, or raise an exception if the
+    database name is not existing.
+
+    Only for internal usage in ``download.py`` and ``query.py``.
+    """
+    if db_name in _db_names.keys():
+        # Convert into E-utility database name
+        return _db_names[db_name]
+    elif db_name in _db_names.values():
+        # Is already E-utility database name
+        return db_name
+    else:
+        raise ValueError("Database '{db_name}' is not existing")

biotite/database/entrez/download.py

@@ -0,0 +1,229 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann"
+__all__ = ["fetch", "fetch_single_file"]
+
+import io
+import os
+from os.path import getsize, isdir, isfile, join
+import requests
+from biotite.database.entrez.check import check_for_errors
+from biotite.database.entrez.dbnames import sanitize_database_name
+from biotite.database.entrez.key import get_api_key
+from biotite.database.error import RequestError
+
+_fetch_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
+
+
+def fetch(
+    uids,
+    target_path,
+    suffix,
+    db_name,
+    ret_type,
+    ret_mode="text",
+    overwrite=False,
+    verbose=False,
+):
+    """
+    Download files from the NCBI Entrez database in various formats.
+
+    The data for each UID will be fetched into a separate file.
+
+    A list of valid database, retrieval type and mode combinations can
+    be found under
+    `<https://www.ncbi.nlm.nih.gov/books/NBK25499/table/chapter4.T._valid_values_of__retmode_and/?report=objectonly>`_
+
+    This function requires an internet connection.
+
+    Parameters
+    ----------
+    uids : str or iterable object of str
+        A single *unique identifier* (UID) or a list of UIDs of the
+        file(s) to be downloaded.
+    target_path : str or None
+        The target directory of the downloaded files.
+        If ``None``, the file content is stored in a file-like object
+        (`StringIO` or `BytesIO`, respectively).
+    suffix : str
+        The file suffix of the downloaded files. This value is
+        independent of the retrieval type.
+    db_name : str:
+        E-utility or common database name.
+    ret_type : str
+        Retrieval type
+    ret_mode : str, optional
+        Retrieval mode
+    overwrite : bool, optional
+        If true, existing files will be overwritten. Otherwise the
+        respective file will only be downloaded if the file does not
+        exist yet in the specified target directory or if the file is
+        empty. (Default: False)
+    verbose: bool, optional
+        If true, the function will output the download progress.
+        (Default: False)
+
+    Returns
+    -------
+    files : str or StringIO or BytesIO or list of (str or StringIO or BytesIO)
+        The file path(s) to the downloaded files.
+        If a single string (a single UID) was given in `uids`,
+        a single string is returned. If a list (or other iterable
+        object) was given, a list of strings is returned.
+        If `target_path` is ``None``, the file contents are stored in
+        either `StringIO` or `BytesIO` objects.
+
+    Warnings
+    --------
+    Even if you give valid input to this function, in rare cases the
+    database might return no or malformed data to you.
+    In these cases the request should be retried.
+    When the issue occurs repeatedly, the error is probably in your
+    input.
+
+    See also
+    --------
+    fetch_single_file
+
+    Examples
+    --------
+
+    >>> import os.path
+    >>> files = fetch(["1L2Y_A","3O5R_A"], path_to_directory, suffix="fa",
+    ...               db_name="protein", ret_type="fasta")
+    >>> print([os.path.basename(file) for file in files])
+    ['1L2Y_A.fa', '3O5R_A.fa']
+    """
+    # If only a single UID is present,
+    # put it into a single element list
+    if isinstance(uids, str):
+        uids = [uids]
+        single_element = True
+    else:
+        single_element = False
+    # Create the target folder, if not existing
+    if target_path is not None and not isdir(target_path):
+        os.makedirs(target_path)
+    files = []
+    for i, id in enumerate(uids):
+        # Verbose output
+        if verbose:
+            print(f"Fetching file {i+1:d} / {len(uids):d} ({id})...", end="\r")
+        # Fetch file from database
+        if target_path is not None:
+            file = join(target_path, id + "." + suffix)
+        else:
+            file = None
+        if file is None or not isfile(file) or getsize(file) == 0 or overwrite:
+            param_dict = {
+                "db": sanitize_database_name(db_name),
+                "id": id,
+                "rettype": ret_type,
+                "retmode": ret_mode,
+                "tool": "Biotite",
+                "mail": "padix.key@gmail.com",
+            }
+            api_key = get_api_key()
+            if api_key is not None:
+                param_dict["api_key"] = api_key
+            r = requests.get(_fetch_url, params=param_dict)
+            content = r.text
+            check_for_errors(content)
+            if content.startswith(" Error"):
+                raise RequestError(content[8:])
+            if file is None:
+                file = io.StringIO(content)
+            else:
+                with open(file, "w+") as f:
+                    f.write(content)
+        files.append(file)
+    if verbose:
+        print("\nDone")
+    # If input was a single ID, return only a single path
+    if single_element:
+        return files[0]
+    else:
+        return files
+
+
+def fetch_single_file(
+    uids, file_name, db_name, ret_type, ret_mode="text", overwrite=False
+):
+    """
+    Almost the same as :func:`fetch()`, but the data for the given UIDs
+    will be stored in a single file.
+
+    Parameters
+    ----------
+    uids : iterable object of str
+        A list of UIDs of the
+        file(s) to be downloaded.
+    file_name : str or None
+        The file path, including file name, to the target file.
+    db_name : str:
+        E-utility or common database name.
+    ret_type : str
+        Retrieval type.
+    ret_mode : str, optional
+        Retrieval mode.
+    overwrite : bool, optional
+        If false, the file is only downloaded, if no file with the same
+        name already exists.
+
+    Returns
+    -------
+    file : str or StringIO or BytesIO
+        The file name of the downloaded file.
+        If `file_name` is ``None``, the file content is stored in
+        either a `StringIO` or a `BytesIO` object.
+
+    Warnings
+    --------
+    Even if you give valid input to this function, in rare cases the
+    database might return no or malformed data to you.
+    In these cases the request should be retried.
+    When the issue occurs repeatedly, the error is probably in your
+    input.
+
+    See also
+    --------
+    fetch
+    """
+    if (
+        file_name is not None
+        and os.path.isfile(file_name)
+        and getsize(file_name) > 0
+        and not overwrite
+    ):
+        # Do no redownload the already existing file
+        return file_name
+    uid_list_str = ""
+    for id in uids:
+        uid_list_str += id + ","
+    # Remove terminal comma
+    uid_list_str = uid_list_str[:-1]
+    param_dict = {
+        "db": sanitize_database_name(db_name),
+        "id": uid_list_str,
+        "rettype": ret_type,
+        "retmode": ret_mode,
+        "tool": "Biotite",
+        "mail": "padix.key@gmail.com",
+    }
+    api_key = get_api_key()
+    if api_key is not None:
+        param_dict["api_key"] = api_key
+    r = requests.get(_fetch_url, params=param_dict)
+    content = r.text
+    check_for_errors(content)
+    if content.startswith(" Error"):
+        raise RequestError(content[8:])
+    if file_name is None:
+        return io.StringIO(content)
+    else:
+        with open(file_name, "w+") as f:
+            f.write(content)
+        return file_name

biotite/database/entrez/key.py

@@ -0,0 +1,44 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann"
+__all__ = ["set_api_key", "get_api_key"]
+
+
+_API_KEY = None
+
+
+def get_api_key():
+    """
+    Get the
+    `NCBI API key <https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/>`_.
+
+    Returns
+    -------
+    api_key : str or None
+        The API key, if it was already set before, ``None`` otherwise.
+    """
+    global _API_KEY
+    return _API_KEY
+
+
+def set_api_key(key):
+    """
+    Set the
+    `NCBI API key <https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/>`_.
+
+    Using an API key increases the request limit on the NCBI servers
+    and is automatically used by functions in
+    :mod:`biotite.database.entrez`.
+    This key is kept only in memory and hence removed in the end of the
+    Python session.
+
+    Parameters
+    ----------
+    api_key : str
+        The API key.
+    """
+    global _API_KEY
+    _API_KEY = key

biotite/database/entrez/query.py

@@ -0,0 +1,262 @@
+# This source code is part of the Biotite package and is distributed
+# under the 3-Clause BSD License. Please see 'LICENSE.rst' for further
+# information.
+
+__name__ = "biotite.database.entrez"
+__author__ = "Patrick Kunzmann"
+__all__ = ["Query", "SimpleQuery", "CompositeQuery", "search"]
+
+import abc
+from xml.etree import ElementTree
+import requests
+from biotite.database.entrez.check import check_for_errors
+from biotite.database.entrez.dbnames import sanitize_database_name
+from biotite.database.entrez.key import get_api_key
+from biotite.database.error import RequestError
+
+_search_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
+
+
+class Query(metaclass=abc.ABCMeta):
+    """
+    Base class for a wrapper around a search term
+    for the NCBI Entrez search service.
+    """
+
+    def __init__(self):
+        pass
+
+    @abc.abstractmethod
+    def __str__(self):
+        pass
+
+    def __or__(self, operand):
+        if not isinstance(operand, Query):
+            operand = SimpleQuery(operand)
+        return CompositeQuery("OR", self, operand)
+
+    def __and__(self, operand):
+        if not isinstance(operand, Query):
+            operand = SimpleQuery(operand)
+        return CompositeQuery("AND", self, operand)
+
+    def __xor__(self, operand):
+        if not isinstance(operand, Query):
+            operand = SimpleQuery(operand)
+        return CompositeQuery("NOT", self, operand)
+
+
+class CompositeQuery(Query):
+    """
+    A representation of an composite query
+    for the NCBI Entrez search service.
+
+    A composite query is a combination of two other queries,
+    combined either with an 'AND', 'OR' or 'NOT' operator.
+
+    Usually the user does not create instances of this class directly,
+    but :class:`Query` instances are combined with
+    ``|`` (OR), ``&`` (AND) or ``^`` (NOT).
+
+    Parameters
+    ----------
+    operator: str, {"AND", "OR", "NOT"}
+        The combination operator.
+    queries : iterable object of SimpleQuery
+        The queries to be combined.
+
+    Examples
+    --------
+
+    >>> query = SimpleQuery("Escherichia coli", "Organism") & \\
+    ... SimpleQuery("90:100", "Sequence Length")
+    >>> print(type(query).__name__)
+    CompositeQuery
+    >>> print(query)
+    ("Escherichia coli"[Organism]) AND (90:100[Sequence Length])
+    """
+
+    def __init__(self, operator, query1, query2):
+        super().__init__()
+        self._op = operator
+        self._q1 = query1
+        self._q2 = query2
+
+    def __str__(self):
+        return "({:}) {:} ({:})".format(str(self._q1), self._op, self._q2)
+
+
+class SimpleQuery(Query):
+    """
+    A simple query for the NCBI Entrez search service without
+    combination via 'AND', 'OR' or 'NOT'. A query consists of a search
+    term and an optional field.
+
+    A list of available search fields with description can be found
+    `here <https://www.ncbi.nlm.nih.gov/books/NBK49540/>`_.
+
+    Parameters
+    ----------
+    term: str
+        The search term.
+    field : str, optional
+        The field to search the term in.
+        The list of possible fields and the required search term
+        formatting can be found
+        `here <https://www.ncbi.nlm.nih.gov/books/NBK49540/>`_.
+        By default the field is omitted and all fields are searched in
+        for the term, implicitly.
+
+    Examples
+    --------
+
+    >>> query = SimpleQuery("Escherichia coli")
+    >>> print(query)
+    "Escherichia coli"
+    >>> query = SimpleQuery("Escherichia coli", "Organism")
+    >>> print(query)
+    "Escherichia coli"[Organism]
+    """
+
+    # Field identifiers are taken from
+    # https://www.ncbi.nlm.nih.gov/books/NBK49540/
+    _fields = [
+        "Accession",
+        "All Fields",
+        "Author",
+        "EC/RN Number",
+        "Feature Key",
+        "Filter",
+        "Gene Name",
+        "Genome Project",
+        "Issue",
+        "Journal",
+        "Keyword",
+        "Modification Date",
+        "Molecular Weight",
+        "Organism",
+        "Page Number",
+        "Primary Accession",
+        "Properties",
+        "Protein Name",
+        "Publication Date",
+        "SeqID String",
+        "Sequence Length",
+        "Substance Name",
+        "Text Word",
+        "Title",
+        "Volume",
+        # Abbreviations
+        "ACCN",
+        "ALL",
+        "AU",
+        "AUTH",
+        "ECNO",
+        "FKEY",
+        "FILT",
+        "SB",
+        "GENE",
+        "ISS",
+        "JOUR",
+        "KYWD",
+        "MDAT",
+        "MOLWT",
+        "ORGN",
+        "PAGE",
+        "PACC",
+        "PORGN",
+        "PROP",
+        "PROT",
+        "PDAT",
+        "SQID",
+        "SLEN",
+        "SUBS",
+        "WORD",
+        "TI",
+        "TITL" "VOL",
+    ]
+
+    def __init__(self, term, field=None):
+        super().__init__()
+        if field is not None:
+            if field not in SimpleQuery._fields:
+                raise ValueError(f"Unknown field identifier '{field}'")
+        for invalid_string in ['"', "AND", "OR", "NOT", "[", "]", "(", ")", "\t", "\n"]:
+            if invalid_string in term:
+                raise ValueError(f"Query contains illegal term {invalid_string}")
+        if " " in term:
+            # Encapsulate in quotes if spaces are in search term
+            term = f'"{term}"'
+        self._term = term
+        self._field = field
+
+    def __str__(self):
+        string = self._term
+        if self._field is not None:
+            string += f"[{self._field}]"
+        return string
+
+
+def search(query, db_name, number=20):
+    r"""
+    Get all PDB IDs that meet the given query requirements,
+    via the NCBI ESearch service.
+
+    This function requires an internet connection.
+
+    Parameters
+    ----------
+    query : Query
+        The search query.
+    db_name : str:
+        E-utility or common database name.
+    number : Query
+        The maximum number of UIDs that are obtained.
+
+    Returns
+    -------
+    ids : list of str
+        A list of strings containing all NCBI UIDs (accession number)
+        that meet the query requirements.
+
+    Warnings
+    --------
+    Even if you give valid input to this function, in rare cases the
+    database might return no or malformed data to you.
+    In these cases the request should be retried.
+    When the issue occurs repeatedly, the error is probably in your
+    input.
+
+    Notes
+    -----
+    A list of available search fields with description can be found
+    `here <https://www.ncbi.nlm.nih.gov/books/NBK49540/>`_.
+
+    Examples
+    --------
+    >>> query = SimpleQuery("Escherichia coli", "Organism") & \
+    ...         SimpleQuery("90:100", "Sequence Length")
+    >>> ids = search(query, "nuccore", number=5)
+    >>> print(ids)
+    ['...', '...', '...', '...', '...']
+    """
+    param_dict = {
+        "db": sanitize_database_name(db_name),
+        "term": str(query),
+        "retmax": str(number),
+    }
+    api_key = get_api_key()
+    if api_key is not None:
+        param_dict["api_key"] = api_key
+    r = requests.get(_search_url, params=param_dict)
+    xml_response = r.text
+    check_for_errors(xml_response)
+    try:
+        root = ElementTree.fromstring(xml_response)
+    except ElementTree.ParseError:
+        if len(xml_response) > 100:
+            xml_response = xml_response[:100] + "..."
+        raise RequestError(f"Invalid server response: {xml_response}")
+    xpath = ".//IdList/Id"
+    uids = [element.text for element in root.findall(xpath)]
+    return uids