napistu 0.1.0__py3-none-any.whl

napistu/ingestion/cpr_edgelist.py

@@ -0,0 +1,106 @@
+"""
+Module with helper functions to deal with edgelists
+
+Edgelists are assumed to be DataFrames whose first two columns represent an Edge relation, eg From, To
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def remove_reciprocal_interactions(
+    edgelist: pd.DataFrame, extra_defining_vars: list = list()
+) -> pd.DataFrame:
+    """Remove reciprocal edges from an edgelist (i.e., if B-A always exists for every A-B then remove B-A)
+
+    Args:
+        edgelist (pd.DataFrame): edgelist (pd.DataFrame): edgelist where the first two
+          columns are assumed to be the edge vertices
+        extra_defining_vars (list): list (which can be empty) of variables which define
+          a unique interaction beyond the vertices
+
+    Returns:
+        indegenerate_edgelist (pd.DataFrame): edgelist with B-A edges removed and A-B retained
+
+    """
+
+    edgelist_vars = edgelist.columns.tolist()[0:2]
+    logger.info(
+        "Removing reciprocal interactions treating "
+        f"{edgelist_vars[0]} and {edgelist_vars[1]} as vertices"
+    )
+
+    reciprocal_interaction_fraction = count_fraction_of_reciprocal_interactions(
+        edgelist, extra_defining_vars
+    )
+    if reciprocal_interaction_fraction != 1:
+        raise ValueError(
+            f"Only {reciprocal_interaction_fraction} of edges are present as reciprocal edges;"
+            " this method of removing reciprocal edges will be unreliable"
+        )
+
+    indegenerate_edgelist = edgelist.loc[
+        edgelist[edgelist_vars[0]] < edgelist[edgelist_vars[1]]
+    ]
+
+    return indegenerate_edgelist
+
+
+def count_fraction_of_reciprocal_interactions(
+    edgelist: pd.DataFrame, extra_defining_vars: list = list()
+) -> float:
+    """Count the fraction of A-B edges which also show up as B-A edges
+
+    Args:
+        edgelist (pd.DataFrame): edgelist (pd.DataFrame): edgelist where the first two
+          columns are assumed to be the edge vertices
+        extra_defining_vars (list): list (which can be empty) of variables which define
+          a unique interaction beyond the vertices
+
+    Returns:
+        fraction (float): fraction of A-B edges which are also included as B-A edges
+
+    """
+
+    # first two variables are assumed to be vertices of edgelist
+    edgelist_vars = edgelist.columns.tolist()[0:2]
+    logger.info(
+        "Counting the fraction of reciprocal interactions treating "
+        f"{edgelist_vars[0]} and {edgelist_vars[1]} as vertices"
+    )
+
+    # extra defining variables must exist
+    missing_extra_defining_vars = set(extra_defining_vars).difference(
+        set(edgelist.columns)
+    )
+    if len(missing_extra_defining_vars) > 0:
+        raise ValueError(
+            f"{', '.join(missing_extra_defining_vars)} are \"extra_defining_vars\" "
+            "but were missing from the edgelist"
+        )
+
+    extended_edgelist_vars = [*edgelist_vars, *extra_defining_vars]
+    logger.info(
+        f"{', '.join(extra_defining_vars)} will be used as \"extra_defining_vars\" "
+        "which much match across reciprocal edges for the edge to be identical"
+    )
+
+    possible_reciprocal_interactions = (
+        edgelist[extended_edgelist_vars]
+        .rename(
+            {edgelist_vars[0]: edgelist_vars[1], edgelist_vars[1]: edgelist_vars[0]},
+            axis=1,
+        )
+        .assign(reciprocal_exists=True)
+    )
+
+    reciprocal_interaction_test = edgelist[extended_edgelist_vars].merge(
+        possible_reciprocal_interactions
+    )
+
+    return reciprocal_interaction_test.shape[0] / edgelist.shape[0]

napistu/ingestion/identifiers_etl.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import os
+import re
+
+import pandas as pd
+import requests
+from napistu.ingestion.constants import IDENTIFIERS_ETL_SBO_URL
+from napistu.ingestion.constants import IDENTIFIERS_ETL_YEAST_FIELDS
+from napistu.ingestion.constants import IDENTIFIERS_ETL_YEAST_URL
+from napistu.ingestion.constants import IDENTIFIERS_ETL_YEAST_HEADER_REGEX
+
+
+def read_yeast_identifiers(url: str = IDENTIFIERS_ETL_YEAST_URL):
+    """Read Yeast Identifiers
+    Generate a pd.DataFrame which maps between yeast identifiers including
+    common and systematic (OLN) names, as well as Swiss-Prot and SGD identifiers.
+
+    Params:
+        url (str): url to the identifier file
+    Returns:
+        pd.DataFrame with one row per gene
+    """
+    response = requests.get(url).text
+
+    yeast_id_list = list()
+    break_line_hit = 0
+    for line in response.splitlines():
+        if re.match(IDENTIFIERS_ETL_YEAST_HEADER_REGEX, line):
+            # find start and end of header indicated by a line of underscores
+            break_line_hit += 1
+            continue
+
+        if break_line_hit >= 2:
+            if line == "":
+                # reached the end
+                break
+
+            # split each line into a list of fields, the only optional field is 3d
+            # all white spaces are space
+            line = re.sub(" +", " ", line)
+            line = re.sub("; ", ";", line)
+            # remove pol and gag designations from transposons since they are an unnecessary extra field
+            line = re.sub("(-[0-9]) (GAG)|(POL)", "\\1", line)
+
+            line = line.split()
+
+            if line[6] != "(3)":
+                # if no 3D field is present then create one
+                line.insert(6, "none")
+
+            # split common fields into a separate list
+            common_list = line[0].split(";")
+            line[0] = common_list[0]
+            line.insert(1, common_list)
+
+            if len(line) != 9:
+                raise ValueError(
+                    "the yeast id file could not be read; all entries should have 8 fields"
+                )
+
+            yeast_id_list.append(dict(zip(IDENTIFIERS_ETL_YEAST_FIELDS, line)))
+
+    return pd.DataFrame(yeast_id_list)
+
+
+def read_sbo_ontology(
+    url: str = IDENTIFIERS_ETL_SBO_URL, verbose: bool = False
+) -> pd.DataFrame:
+    """Read SBO Ontology
+    Read the Systems Biology Ontology (SBO) identifiers and reformat the obo results into a pd.DataFrame.
+
+    Params:
+        url (str): url to the obo specification file
+        verbose (bool): throw warnings when attributes are overwritten
+    Returns:
+        pd.DataFrame
+    """
+
+    # save the obo file locally
+    tmp_file = os.path.join("/tmp", "sbo.obo")
+    r = requests.get(url, allow_redirects=True)
+    open(tmp_file, "wb").write(r.content)
+
+    with open(tmp_file) as sbo:
+        sbo_dict = dict()
+        current_id = None
+        in_header = True
+        for line in sbo:
+            # skip the header
+            if line == "[Term]\n":
+                in_header = False
+                continue
+            if in_header:
+                continue
+
+            line_entries = line.split(":", 1)
+
+            if len(line_entries) == 2:
+                entry_type = line_entries[0]
+                entry_value = line_entries[1].strip()
+
+                # drop type defs
+                if (
+                    (current_id is not None)
+                    and (entry_type != "id")
+                    and (re.match("SBO", current_id) is None)
+                ):
+                    continue
+
+                # clean-up definitions
+                if entry_type == "is_a":
+                    entry_value = re.match("SBO:[0-9]+", entry_value)[0]
+
+                # if a new id has been reached then initilize a new dict and
+                # update current id
+
+                if entry_type == "id":
+                    current_id = entry_value
+                    if re.match("SBO", current_id) is not None:
+                        sbo_dict[current_id] = {"is_a": []}
+                    continue
+
+                if entry_type == "is_a":
+                    sbo_dict[current_id]["is_a"].append(entry_value)
+                else:
+                    # add a new entry
+                    if (entry_type in sbo_dict[current_id].keys()) and verbose:
+                        print(
+                            f"2+ {entry_type} entries were found for {current_id}, only one value should be present "
+                        )
+                    sbo_dict[current_id][entry_type] = entry_value
+
+    sbo_df = pd.DataFrame(sbo_dict).T
+
+    obsolete_terms = set(
+        sbo_df["name"][sbo_df["name"].str.match("obsolete")].index.tolist()
+    )
+    sbo_df["is_obsolete"] = [
+        (x in obsolete_terms) | (len(set(y).intersection(obsolete_terms)) > 0)
+        for x, y in zip(sbo_df.index, sbo_df["is_a"])
+    ]
+
+    sbo_df = sbo_df[["name", "comment", "is_a", "is_obsolete"]]
+    sbo_df.index.name = "sbo_term"
+    sbo_df = sbo_df.reset_index()
+
+    return sbo_df

napistu/ingestion/obo.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import collections
+import os
+from itertools import chain
+from typing import Any
+
+import igraph as ig
+import pandas as pd
+from napistu import utils
+from napistu.ingestion.constants import OBO_GO_BASIC_LOCAL_TMP
+from napistu.ingestion.constants import OBO_GO_BASIC_URL
+
+
+def create_go_parents_df(go_basic_obo_df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Create the GO Parents Table
+
+    Reformat a table with GO attributes into a table with child-parent relationships
+
+    Args:
+        go_basic_obo_df (pd.DataFrame): Table generated from parsing go-basic.obo with
+            obo.format_obo_dict_as_df
+
+    Returns:
+        go_parents_df (pd.DataFrame): a table with:
+          - parent_id: GO ID of parent (from an is-a entry)
+          - parent_name: common name of parent (from an is-a entry)
+          - child_id: GO ID from the index
+
+    """
+    # filter to CC ontology and look at a series
+    # where the index is GO IDs and values is a list of parent "is-a" relations
+    cc_parents = go_basic_obo_df.query("namespace == 'cellular_component'")["is_a"]
+
+    # this is currently at 4496 rows - this is expected to slowly increase
+    assert cc_parents.shape[0] >= 4496
+    assert cc_parents.shape[0] < 5000
+
+    # convert from a list of strings to a list of dicts then expand so each
+    # dict is its own row
+    parent_entries = cc_parents.map(_isa_str_list_to_dict_list).explode()
+    # drop orphans which will be NaN's after the explosion
+    parent_entries = parent_entries[~parent_entries.isnull()]
+
+    # convert to a DF which just has string variables
+    go_parents_df = pd.DataFrame(parent_entries.tolist())
+    go_parents_df["child_id"] = parent_entries.index
+
+    # currently at 4688 rows - this may increase or decrease but will do so slowly
+    assert go_parents_df.shape[0] > 4600
+    assert go_parents_df.shape[0] < 5000
+
+    return go_parents_df
+
+
+def read_obo_as_dict(local_obo_path: str) -> dict:
+    """
+    Read OBO as Dictionary
+
+    The Open Biological and Biomedical Ontologies (OBO) format is a standard format
+    for representing ontologies. Many parsers exist for obo but since we are not
+    relying extensively on it and we are trying to minimize dependencies here we provide a
+    few functions for parsing standard obo formats.
+
+    Args:
+        local_obo_path (str): path to a local obo file.
+
+    Returns
+        term_dict (dict): dictionary where keys are ids and values are tuples
+            containing (attribute, value) pairs
+    """
+    # create a dict where keys are term IDs and values are lists of tuples
+    term_dict = dict()  # type: dict[str, Any]
+    term_is_next = False
+    active_term = None
+
+    with open(local_obo_path) as file:
+        for line in file:
+            line_strip = line.rstrip()
+
+            # reset the active term using the break between term definitions
+            if line_strip == "":
+                active_term = None
+
+            line_as_tuple = _format_entry_tuple(line_strip)
+
+            # catch new term definitions
+            if term_is_next:
+                attrib, value = line_as_tuple
+                if attrib != "id":
+                    raise ValueError(
+                        f'{line_strip} was expected to be an "id" but it was not recongized as one'
+                    )
+
+                active_term = value
+                term_dict[active_term] = list()
+                term_is_next = False
+                continue
+
+            if line_strip == "[Term]":
+                term_is_next = True
+                continue
+            else:
+                term_is_next = False
+
+            if active_term is not None:
+                term_dict[active_term].append(line_as_tuple)
+
+    return term_dict
+
+
+def format_obo_dict_as_df(obo_term_dict: dict) -> pd.DataFrame:
+    """
+    Format an OBO Dict as a DataFrame
+
+    Reorganize a dictionary of tuples into a DataFrame
+
+    Args:
+        term_dict (dict): dictionary where keys are ids and values are tuples
+            containing (attribute, value) pairs
+
+    Returns
+        obo_df (pd.DataFrame): A pd.DataFrame with one row per identifier and one columns for unique attribute
+    """
+    # find attributes which can occur multiple times. These will be represented as lists within the
+    # pandas DataFrame. The remaining attributes will just be strings.
+    dups = [_find_obo_attrib_dups(obo_term_dict[k]) for k in obo_term_dict.keys()]
+    degenerate_attribs = set(chain(*dups))
+
+    # reorganize term as list to setup creation of pd.DataFrame
+    term_dicts = list()
+    for k, v in obo_term_dict.items():
+        term_dict = _reformat_obo_entry_as_dict(v, degenerate_attribs)
+        term_dict["id"] = k
+        term_dicts.append(term_dict)
+
+    obo_df = pd.DataFrame(term_dicts).set_index("id")
+
+    return obo_df
+
+
+def _reformat_obo_entry_as_dict(one_term, degenerate_attribs) -> dict:
+    term_dict = dict()
+    for attrib in degenerate_attribs:
+        term_dict[attrib] = list()
+
+    for attrib, value in one_term:
+        if attrib in degenerate_attribs:
+            term_dict[attrib].append(value)
+        else:
+            term_dict[attrib] = value
+
+    return term_dict
+
+
+def create_parent_child_graph(go_parents_df: pd.DataFrame) -> ig.Graph:
+    """
+    Create Parent:Child Graph
+
+    Format the Simple GO CC Ontology as a Directed Acyclic Graph (DAG).
+
+    Args:
+        go_parents_df (pd.DataFrame): a table with:
+          - parent_id: GO ID of parent (from an is-a entry)
+          - parent_name: common name of parent (from an is-a entry)
+          - child_id: GO ID from the index
+
+    Returns:
+        parent_child_graph (ig.Graph): a DAG formed from parent-child relationships.
+
+    """
+    valid_go_ids = {
+        *go_parents_df["parent_id"].tolist(),
+        *go_parents_df["child_id"].tolist(),
+    }
+    valid_go_ids_df = pd.DataFrame(valid_go_ids)
+    valid_go_ids_df.columns = ["go_id"]  # type: ignore
+
+    # format edgelist as an igraph network
+    parent_child_graph = ig.Graph.DictList(
+        vertices=valid_go_ids_df.to_dict("records"),
+        edges=go_parents_df[["child_id", "parent_id"]].to_dict("records"),
+        directed=True,
+        vertex_name_attr="go_id",
+        edge_foreign_keys=("child_id", "parent_id"),
+    )
+
+    # is it a fully connected DAG as expected?
+    assert parent_child_graph.is_dag()
+    assert parent_child_graph.is_connected("weak")
+
+    return parent_child_graph
+
+
+def create_go_ancestors_df(parent_child_graph: ig.Graph) -> pd.DataFrame:
+    """
+    Create GO Ancestors DataFrame
+
+    Args:
+        parent_child_graph (ig.Graph): a DAG formed from parent-child relationships.
+
+    Returns:
+        go_ancestors_df (pd.DataFrame): a table with:
+          - go_id: GO ID of a CC GO term of interest
+          - ancestor_id: An ancestor (parent, parent of parent, ...)'s GO CC ID
+    """
+    # find the ancestors of each vertex
+    ancestor_dict = [
+        {
+            "go_id": v["go_id"],
+            "ancestor_id": parent_child_graph.vs(
+                parent_child_graph.subcomponent(v, mode=ig.OUT)
+            ).get_attribute_values("go_id"),
+        }
+        for v in parent_child_graph.vs
+    ]
+
+    go_ancestors_df = pd.DataFrame(ancestor_dict).explode("ancestor_id")
+    # drop self edges
+    go_ancestors_df = go_ancestors_df[
+        go_ancestors_df["go_id"] != go_ancestors_df["ancestor_id"]
+    ]
+
+    return go_ancestors_df
+
+
+def _download_go_basic_obo(local_obo_path: str = OBO_GO_BASIC_LOCAL_TMP) -> None:
+    """Download an OBO file containing GO categories and their relations (but not the genes in each category)."""
+
+    utils.download_wget(OBO_GO_BASIC_URL, local_obo_path)
+
+    if not os.path.isfile(local_obo_path):
+        raise FileNotFoundError(
+            f"{local_obo_path} was not found after trying to download from {OBO_GO_BASIC_URL}"
+        )
+
+
+def _isa_str_list_to_dict_list(isa_list: list) -> list[dict[str, Any]]:
+    """Split parent-child relationships from individual strings to dictionaries where parent and child are separated."""
+
+    split_vals = [tuple(val.split(" ! ")) for val in isa_list]
+
+    isa_dict_list = list()
+    for split_val in split_vals:
+        assert len(split_val) == 2
+
+        isa_dict_list.append({"parent_id": split_val[0], "parent_name": split_val[1]})
+
+    return isa_dict_list
+
+
+def _format_entry_tuple(line_str: str) -> tuple | None:
+    """Split and return a colon-separated tuple."""
+
+    entry = line_str.split(": ", maxsplit=1)
+    if len(entry) == 2:
+        return tuple(entry)
+    return None
+
+
+def _find_obo_attrib_dups(one_term) -> list:
+    """Identify attributes which are present multiple times."""
+
+    attrib_count = collections.Counter([v[0] for v in one_term])
+    duplicated_attributes = [item for item, count in attrib_count.items() if count > 1]
+
+    return duplicated_attributes