gesel 0.1.0__py3-none-any.whl

gesel/__init__.py

@@ -0,0 +1,48 @@
+import sys
+
+if sys.version_info[:2] >= (3, 8):
+    # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8`
+    from importlib.metadata import PackageNotFoundError, version  # pragma: no cover
+else:
+    from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError
+
+
+from ._download_database_file import download_database_file, database_url
+from ._download_database_ranges import download_database_ranges, download_multipart_ranges, range_concurrency
+from ._download_gene_file import download_gene_file, gene_url
+
+from ._fetch_all_collections import fetch_all_collections
+from ._fetch_all_genes import fetch_all_genes
+from ._fetch_all_sets import fetch_all_sets
+from ._fetch_some_sets import fetch_some_sets, fetch_set_sizes
+from ._fetch_some_collections import fetch_some_collections, fetch_collection_sizes
+
+from ._fetch_genes_for_all_sets import fetch_genes_for_all_sets
+from ._fetch_sets_for_all_genes import fetch_sets_for_all_genes
+from ._fetch_genes_for_some_sets import fetch_genes_for_some_sets
+from ._fetch_sets_for_some_genes import fetch_sets_for_some_genes, effective_number_of_genes
+from ._find_overlapping_sets import find_overlapping_sets
+
+from ._map_genes_by_name import map_genes_by_name
+from ._search_genes import search_genes
+from ._search_set_text import search_set_text
+
+from ._new_config import new_config, flush_memory_cache
+from ._utils import cache_directory
+
+
+__all__ = []
+_toignore = set(["sys", "biocutils", "biocframe"]) 
+for _name in dir():
+    if _name.startswith("_") or _name in _toignore:
+        continue
+    __all__.append(_name)

gesel/_download_database_file.py

@@ -0,0 +1,79 @@
+from typing import Optional
+from . import _utils as utils
+
+
+def download_database_file(
+    name: str,
+    url: Optional[str] = None,
+    cache: Optional[str] = None,
+    overwrite: bool = False
+) -> str:
+    """
+    Download Gesel database files and store them in a cache on the local file system.
+
+    Args:
+        name:
+            Name of the file.
+            This usually has the species identifier as a prefix.
+
+        url:
+            Base URL to the Gesel database files.
+            If ``None``, it is set to :py:func:`~database_url`.
+
+        cache:
+            Path to a cache directory.
+            If ``None``, it is set to :py:func:`~cache_directory`.
+
+        overwrite:
+            Boolean indicating whether any cached file should be overwritten with a new download.
+
+    Returns:
+        Path to the downloaded file on the local file system.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.download_database_file("9606_collections.tsv.gz")
+    """
+    if url is None:
+        url = database_url()
+    return utils._download_file(cache, url + "/" + name, overwrite)
+
+
+_db_url = None
+
+
+def database_url(url: Optional[str] = None) -> str:
+    """
+    Get or set the base URL to the Gesel database files, which is used in :py:func:`~download_database_file`.
+
+    Args:
+        url: 
+            The new database URL.
+
+    Returns:
+        If ``url = None``, the URL to the Gesel database is returned.
+        The URL defaults to the `GitHub releases page <https://github.com/LTLA/gesel-feedstock>`_;
+        this can be altered by setting the ``GESEL_DATABASE_URL`` environment variable before the first call to this function.
+
+        If ``url`` is provided, this function sets the database URL to ``url``, and returns the previous value of the URL.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.database_url()
+        >>> old = gesel.database_url("https://foo.bar")
+        >>> gesel.database_url(old)
+        >>> gesel.database_url()
+    """
+
+    global _db_url
+    if _db_url is None:
+        import os
+        if "GESEL_DATABASE_URL" in os.environ:
+            _db_url = os.environ["GESEL_DATABASE_URL"]
+        else:
+            _db_url = "https://github.com/LTLA/gesel-feedstock/releases/download/indices-v0.2.1"
+
+    previous = _db_url
+    if url is not None:
+        _db_url = url
+    return previous

gesel/_download_database_ranges.py

@@ -0,0 +1,254 @@
+from ._download_database_file import database_url
+from . import _utils as utils
+from typing import Optional
+import requests
+
+
+def download_database_ranges(
+    name: str,
+    start: list[int],
+    end: list[int],
+    url: Optional[str] = None,
+    multipart: bool = False,
+    concurrency: Optional[int] = None 
+) -> list[str]:
+    """
+    Download any number of byte ranges from a Gesel database file.
+
+    Args:
+        name:
+            Name of the file.
+            This usually has the species identifier as a prefix.
+
+        start:
+            List of integers containing the zero-indexed closed start of each byte range to extract from the file.
+            This list may be empty.
+
+        end:
+            List of integers containing the zero-indexed open end of each byte range to extract from the file.
+            This should have the same length as ``start`` such that the ``i``-th range is defined as ``[start[i], end[i])``.
+            All ranges supplied in a single call to this function should be non-overlapping.
+
+        url:
+            Base URL to the Gesel database files.
+            If ``None``, it is set to :py:func:`~gesel.database_url`.
+
+        multipart:
+            Whether the server at ``url`` supports multi-part range requests.
+
+        concurrency:
+            Maximum number of concurrent range requests.
+            If ``None``, defaults to :py:func:`~gesel.range_concurrency`.
+            Only used if ``multipart = False``.
+
+    Returns:
+        List of byte strings containing the requested bytes for each range.
+        For ranges where ``end <= start``, an empty string is returned.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.download_database_ranges("9606_set2gene.tsv", [0], [100])
+        >>> gesel.download_database_ranges("9606_set2gene.tsv", [10, 100, 1000], [20, 150, 1100])
+    """
+
+    if url is None:
+        url = database_url()
+    url = url + "/" + name
+
+    if multipart:
+        return download_multipart_ranges(url, start, end)
+
+    if concurrency is None:
+        concurrency = range_concurrency()
+    from concurrent.futures import ThreadPoolExecutor
+
+    with ThreadPoolExecutor(max_workers = concurrency) as executor:
+        collected = []
+        for i in range(len(start)):
+            curstart = start[i]
+            curend = end[i]
+            if curstart < curend:
+                collected.append((i, executor.submit(_make_range_request, url, curstart, curend)))
+
+        output = [b""] * len(start)
+        for i, val in collected:
+            output[i] = val.result()
+
+    return output
+
+
+_range_concurrency = 10
+
+
+def range_concurrency(concurrency: Optional[int] = None):
+    """
+    Set the default number of threads for concurrent range requests in :py:func:`~gesel.download_database_ranges`.
+
+    Args:
+        concurrency:
+            Number of threads.
+
+    Results:
+        If ``concurrency = None``, the number of threads is returned (default 10).
+
+        If ``concurrency`` is provided, it is used to set the number of threads, and the previous number of threads is returned.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.range_concurrency()
+        >>> old = gesel.range_concurrency(5)
+        >>> print(old)
+        >>> gesel.range_concurrency(old)
+    """
+    global _range_concurrency
+    previous = _range_concurrency
+    if concurrency is not None:
+        _range_concurrency = concurrency
+    return previous
+
+
+def _make_range_request(url, curstart, curend) -> bytes:
+    resp = requests.get(url, headers = { "Range": "bytes=" + str(curstart) + "-" + str(curend - 1) }) # byte ranges are closed intervals, not half-open.
+    if resp.status_code >= 300:
+        raise utils.format_error(resp)
+    return resp.content[:(curend - curstart)] # We use 'content' instead of 'text' to handle multi-byte characters correctly.
+
+
+def download_multipart_ranges(
+    url: str,
+    start: list[int],
+    end: list[int],
+    _mock = None
+) -> list[str]:
+    """
+    Perform a multi-part range request on a Gesel database file.
+
+    Args:
+        url:
+            URL to a specific Gesel database file.
+
+        start:
+            List of integers containing the zero-indexed closed start of each byte range to extract from the file.
+            This list may be empty.
+
+        end:
+            List of integers containing the zero-indexed open end of each byte range to extract from the file.
+            This should have the same length as ``start`` such that the ``i``-th range is defined as ``[start[i], end[i])``.
+            All ranges supplied in a single call to this function should be non-overlapping.
+
+        _mock:
+            Internal use only.
+
+    Returns:
+        List of byte strings containing the requested bytes for each range.
+        For ranges where ``end <= start``, an empty string is returned.
+
+    Examples:
+        >>> import gesel
+        >>> url = gesel.database_url() + "/9606_set2gene.tsv" 
+        >>> gesel.download_multipart_ranges(url, [0], [100])
+        >>> gesel.download_multipart_ranges(url, [10, 100, 1000], [10, 150, 900])
+        >>> # Note: as of writing, GitHub releases don't support multi-part range requests.
+    """
+
+    collected = []
+    for i in range(len(start)):
+        curstart = start[i]
+        curend = end[i]
+        if curend <= curstart:
+            continue
+        collected.append((curstart, curend, i))
+    collected.sort()
+
+    output = [b""] * len(start)
+    if len(collected) == 0:
+        return output
+
+    formatted = []
+    for curstart, curend, _ in collected:
+        formatted.append(str(curstart) + "-" + str(curend - 1)) # byte ranges are closed intervals, not half-open.
+
+    if _mock is None:
+        resp = requests.get(url, headers = { "Range": "bytes=" + ", ".join(formatted) })
+    else:
+        resp = _mock(url, "bytes=" + ", ".join(formatted)) 
+    if resp.status_code >= 300:
+        raise utils.format_error(resp)
+
+    if len(collected) == 1:
+        curstart, curend, curi = collected[0]
+        output[curi] = resp.content[:(curend - curstart)]
+        return output
+
+    ct = resp.headers["Content-Type"]
+    prefix = "multipart/byteranges; boundary="
+    if not ct.startswith(prefix):
+        raise RuntimeError("unexpected content type from multi-part range request")
+    boundary = ct[len(prefix):]
+
+    # Remember, we operate on the byte strings to correctly handle ranges for multi-byte characters.
+    parsed_start, parsed_content = _parse_multipart_response(resp.content, boundary.encode("utf-8"))
+    import bisect
+    for curstart, curend, curi in collected:
+        chosen = bisect.bisect_right(parsed_start, curstart) - 1
+        offset = parsed_start[chosen]
+        output[curi] = parsed_content[chosen][(curstart - offset):(curend - offset)]
+
+    return output
+
+
+# WARNING: this part only works when the contents of body are fully string-able;
+# otherwise, if it's binary data, we really should be working with them as raw data.
+def _parse_multipart_response(body: bytes, boundary: bytes) -> tuple[list, list]:
+    position = 0
+    boundary = b"--" + boundary
+    starts = []
+    contents = []
+
+    while True:
+        if body[position:(position + len(boundary))] != boundary:
+            raise RuntimeError("missing boundary in multi-part range request")
+        position += len(boundary)
+
+        remaining = body[position:(position + 2)]
+        if remaining == b"--":
+            break 
+        if remaining != b"\r\n":
+            raise RuntimeError("unexpected content range format in multi-part response")
+        position += 2
+
+        # Running through all headers. 
+        headers = {}
+        while True:
+            crlf = body.find(b"\r\n", position)
+            if crlf == -1:
+                raise RuntimeError("failed to find CRLF to terminate headers in multi-part response")
+            curline = body[position:crlf]
+            position = crlf + 2
+            if curline == b"": # headers terminate at a CRLF empty line.
+                break 
+            at_colon = curline.find(b":")
+            headers[curline[:at_colon].lower()] = curline[(at_colon+1):]
+
+        if b"content-range" not in headers:
+            raise RuntimeError("expected a 'content-range' header in each multi-part response")
+        crange = headers[b"content-range"]
+        to_bytes = crange.find(b"bytes ")
+        to_dash = crange.find(b"-")
+        to_slash = crange.find(b"/")
+        if to_bytes == -1 or to_dash == -1 or to_slash == -1 or to_dash >= to_slash:
+            raise RuntimeError("expected 'content-range' header to follow a 'bytes <start>-<end>/<size>' format")
+
+        first = int(crange[(to_bytes + 6):to_dash])
+        last = int(crange[(to_dash + 1):to_slash])
+        starts.append(first) 
+        curlen = last - first + 1
+        contents.append(body[position:(position + curlen)])
+        position += curlen 
+
+        remaining = body[position:(position + 2)]
+        if remaining != b"\r\n":
+            raise RuntimeError("expected a newline after each part of a multi-part response")
+        position += 2
+
+    return starts, contents

gesel/_download_gene_file.py

@@ -0,0 +1,79 @@
+from typing import Optional
+from . import _utils as utils
+
+
+def download_gene_file(
+    name: str,
+    url: Optional[str] = None,
+    cache: Optional[str] = None,
+    overwrite: bool = False
+) -> str:
+    """
+    Download Gesel gene files and store them in a cache on the local file system.
+
+    Args:
+        name:
+            Name of the file.
+            This usually has the species identifier as a prefix.
+
+        url:
+            Base URL to the Gesel gene files.
+            If ``None``, it is set to :py:func:`~gene_url`.
+
+        cache:
+            Path to a cache directory.
+            If ``None``, it is set to :py:func:`~cache_directory`.
+
+        overwrite:
+            Boolean indicating whether any cached file should be overwritten with a new download.
+
+    Returns:
+        Path to the downloaded file on the local file system.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.download_gene_file("9606_symbol.tsv.gz")
+    """
+    if url is None:
+        url = gene_url()
+    return utils._download_file(cache, url + "/" + name, overwrite)
+
+
+_gene_url = None
+
+
+def gene_url(url: Optional[str] = None) -> str:
+    """
+    Get or set the base URL to the Gesel gene files, which is used in :py:func:`~download_gene_file`.
+
+    Args:
+        url: 
+            The new gene URL.
+
+    Returns:
+        If ``url = None``, the URL to the Gesel gene is returned.
+        The default gene URL is set to the `GitHub releases page <https://github.com/LTLA/gesel-feedstock>`_.
+        This can be altered by setting the ``GESEL_GENE_URL`` environment variable.
+
+        If ``url`` is provided, this function sets the gene URL to ``url``, and returns the previous value of the URL.
+
+    Examples:
+        >>> import gesel
+        >>> gesel.gene_url()
+        >>> old = gesel.gene_url("https://foo.bar")
+        >>> gesel.gene_url(old)
+        >>> gesel.gene_url()
+    """
+
+    global _gene_url
+    if _gene_url is None:
+        import os
+        if "GESEL_GENE_URL" in os.environ:
+            _gene_url = os.environ["GESEL_GENE_URL"]
+        else:
+            _gene_url = "https://github.com/LTLA/gesel-feedstock/releases/download/genes-v1.0.0"
+
+    previous = _gene_url
+    if url is not None:
+        _gene_url = url
+    return previous

gesel/_fetch_all_collections.py

@@ -0,0 +1,79 @@
+from typing import Optional
+import biocframe
+from . import _utils as utils
+from . import _new_config as cfg 
+
+
+def fetch_all_collections(species: str, config: Optional[dict] = None) -> biocframe.BiocFrame:
+    """
+    Fetch information about all gene set collections in the Gesel database.
+
+    Args:
+        species:
+            NCBI taxonomy ID of the species of interest.
+
+        config:
+            Configuration object, typically created by :py:func:`~gesel.new_config`.
+            If ``None``, the default configuration is used.
+
+    Returns:
+        A :py:class:`~biocframe.BiocFrame` where each row represents a collection.
+        This contains the following columns:
+
+        - ``title``, string containing the title of the collection.
+        - ``description``, string containing a description of the collection.
+        - ``maintainer``, string containing the identity of the collection's maintainer.
+        - ``source``, string containing the source of origin of the collection.
+        - ``start``, integer containing the set index of the first gene set in this collection.
+          The set index refers to a row in the data frame returned by :py:func:`~gesel.fetch_all_sets`.
+        - ``size``, integer specifying the number of gene sets in the collection.
+
+        If this function is called once, the data frame will be cached in memory and re-used in subsequent calls.
+        The cached information will also be used to speed up :py:func:`~gesel.fetch_some_collections`.
+
+    Examples:
+        >>> import gesel
+        >>> df = gesel.fetch_all_collections("9606")
+        >>> print(df)
+    """
+
+    config = cfg._get_config(config)
+    candidate = cfg._get_cache(config, "fetch_all_collections", species)
+    if candidate is not None:
+        return candidate
+
+    fname = species + "_collections.tsv.gz"
+    path = cfg._fetch_file(config, fname)
+
+    title = []
+    desc = []
+    maintainer = []
+    src = []
+    start = []
+    size = []
+    last = 0
+
+    import gzip
+    with gzip.open(path, "rb") as f:
+        for line in f:
+            details = line.decode("utf-8").split("\t")
+            title.append(details[0])
+            desc.append(details[1])
+            maintainer.append(details[3])
+            src.append(details[4])
+            start.append(last)
+            cursize = int(details[5])
+            size.append(cursize)
+            last += cursize
+
+    output = biocframe.BiocFrame({
+        "title": title,
+        "description": desc,
+        "maintainer": maintainer,
+        "source": src,
+        "start": start,
+        "size": size
+    })
+
+    cfg._set_cache(config, "fetch_all_collections", species, output)
+    return output

gesel/_fetch_all_genes.py

@@ -0,0 +1,70 @@
+from . import _new_config as cfg
+from typing import Optional
+import biocframe
+
+
+def fetch_all_genes(
+    species: str,
+    types: list = ["symbol", "entrez", "ensembl"],
+    config: Optional[dict] = None
+) -> biocframe.BiocFrame:
+    """
+    Fetch names and identifiers of various types for all genes.
+
+    Args:
+        species:
+            NCBI taxonomy ID of the species of interest.
+
+        types:
+            Types of gene names to return.
+            Typically one or more of ``symbol``, ``entrez``, and/or ``ensembl``.
+
+        config:
+            Configuration object, typically created by :py:func:`~gesel.new_config`.
+            If ``None``, the default configuration is used.
+
+    Returns:
+        A :py:class:`~biocframe.BiocFrame` where each row represents a gene.
+        Each column corresponds to one of the ``types`` and is a list of lists.
+        Each inner list in the column contains the names of the specified type for each gene.
+
+    Examples:
+        >>> import gesel
+        >>> df = gesel.fetch_all_genes("9606")
+        >>> print(df)
+        >>> print(df["symbol"][1:10])
+    """
+
+    config = cfg._get_config(config)
+    cached = cfg._get_cache(config, "fetch_all_genes", species)
+
+    modified = False
+    if cached is None:
+        cached = {}
+
+    output = {}
+    for t in types:
+        if t in cached:
+            output[t] = cached[t]
+            continue
+
+        path = cfg._fetch_gene(config, species + "_" + t + ".tsv.gz")
+        collected = []
+
+        import gzip
+        with gzip.open(path, "rb") as f:
+            for line in f:
+                line = line.rstrip()
+                if line == b"":
+                    collected.append([])
+                else:
+                    collected.append(line.decode("utf-8").split("\t"))
+
+        output[t] = collected
+        cached[t] = collected
+        modified = True
+
+    if modified:
+        cfg._set_cache(config, "fetch_all_genes", species, cached)
+
+    return biocframe.BiocFrame(output)

gesel/_fetch_all_sets.py

@@ -0,0 +1,81 @@
+from typing import Optional
+from . import _new_config as cfg
+from ._fetch_all_collections import fetch_all_collections
+import biocframe
+
+
+def fetch_all_sets(species: str, config: Optional[dict] = None) -> biocframe.BiocFrame:
+    """
+    Fetch information about all gene sets in the Gesel database.
+
+    Args:
+        species:
+            NCBI taxonomy ID of the species of interest.
+
+        config:
+            Configuration object, typically created by :py:func:`~gesel.new_config`.
+            If ``None``, the default configuration is used.
+
+    Returns:
+        A :py:class:`~biocframe.BiocFrame` where each row represents a gene set.
+        This contains the following columns:
+
+        - ``name``, string containing the name of the gene set.
+        - ``description``, string containing a description of the gene set.
+        - ``size``: integer specifying the number of genes in this gene set.
+        - ``collection``: integer, the collection index of the collection that contains this gene set.
+          The collection index refers to a row of the data frame returned by :py:func:`~gesel.fetch_all_collections`.
+        - ``number``: integer, the position of the gene set inside the specified collection.
+          The set index of the current gene set is defined by adding ``number`` to the collection's ``start``. 
+
+        If this function is called once, the data frame will be cached in memory and re-used in subsequent calls.
+        The cached information will also be used to speed up :py:func:`~gesel.fetch_some_sets`.
+
+    Examples:
+        >>> import gesel
+        >>> df = gesel.fetch_all_sets("9606")
+        >>> print(df)
+    """
+
+    config = cfg._get_config(config)
+    candidate = cfg._get_cache(config, "fetch_all_sets", species)
+    if candidate is not None:
+        return candidate
+
+    fname = species + "_sets.tsv.gz"
+    path = cfg._fetch_file(config, fname)
+
+    name = []
+    desc = []
+    size = []
+
+    import gzip
+    with gzip.open(path, "rb") as f:
+        for line in f:
+            details = line.rstrip().decode("utf-8").split("\t")
+            name.append(details[0])
+            desc.append(details[1])
+            size.append(int(details[2]))
+
+    info = fetch_all_collections(species, config=config)
+    collection, number = _compute_set_to_collection_indices(info["size"])
+
+    output = biocframe.BiocFrame({
+        "name": name,
+        "description": desc,
+        "size": size,
+        "collection": collection,
+        "number": number
+    })
+
+    cfg._set_cache(config, "fetch_all_sets", species, output)
+    return output
+
+
+def _compute_set_to_collection_indices(collection_sizes: list) -> tuple:
+    collection = []
+    number = []
+    for cidx, csize in enumerate(collection_sizes):
+        collection += [cidx] * csize
+        number += list(range(csize))
+    return collection, number