napistu 0.1.0__py3-none-any.whl

napistu/sbml_dfs_utils.py

@@ -0,0 +1,304 @@
+from __future__ import annotations
+
+import copy
+
+import logging
+import re
+from typing import Any
+from typing import Iterable
+from fs import open_fs
+
+import numpy as np
+import pandas as pd
+from napistu import utils
+from napistu import indices
+
+from napistu.constants import SBML_DFS
+from napistu.constants import IDENTIFIERS
+from napistu.constants import BQB_DEFINING_ATTRS
+from napistu.constants import BQB_DEFINING_ATTRS_LOOSE
+
+logger = logging.getLogger(__name__)
+
+
+def unnest_identifiers(id_table: pd.DataFrame, id_var: str) -> pd.DataFrame:
+    """
+    Unnest Identifiers
+
+    Take a pd.DataFrame containing an array of Identifiers and
+    return one-row per identifier.
+
+    Parameters:
+    id_table: pd.DataFrame
+        a table containing an array of Identifiers
+    id_var: str
+        variable containing Identifiers
+
+    Returns:
+    pd.Dataframe containing the index of id_table but expanded
+    to include one row per identifier
+
+    """
+
+    # validate inputs
+    utils.match_pd_vars(id_table, {id_var}).assert_present()
+
+    N_invalid_ids = sum(id_table[id_var].isna())
+    if N_invalid_ids != 0:
+        raise ValueError(
+            f'{N_invalid_ids} entries in "id_table" were missing',
+            "entries with no identifiers should still include an Identifiers object",
+        )
+
+    # Get the identifier as a list of dicts
+    df = id_table[id_var].apply(lambda x: x.ids if len(x.ids) > 0 else 0).to_frame()
+    # Filter out zero length lists
+    df = df.query(f"{id_var} != 0")
+    # Unnest the list of dicts into one dict per row
+    df = df.explode(id_var)
+    # Unnest the dict into a dataframe
+    df = pd.DataFrame(df[id_var].values.tolist(), index=df.index)
+    # Add the entry number as an index
+    df["entry"] = df.groupby(df.index).cumcount()
+    df.set_index("entry", append=True, inplace=True)
+    return df
+
+
+def id_formatter(id_values: Iterable[Any], id_type: str, id_len: int = 8) -> list[str]:
+    id_prefix = utils.extract_regex_match("^([a-zA-Z]+)_id$", id_type).upper()
+    return [id_prefix + format(x, f"0{id_len}d") for x in id_values]
+
+
+def id_formatter_inv(ids: list[str]) -> list[int]:
+    """
+    ID Formatter Inverter
+
+    Convert from internal IDs back to integer IDs
+    """
+
+    id_val = list()
+    for an_id in ids:
+        if re.match("^[A-Z]+[0-9]+$", an_id):
+            id_val.append(int(re.sub("^[A-Z]+", "", an_id)))
+        else:
+            id_val.append(np.nan)  # type: ignore
+
+    return id_val
+
+
+def get_current_max_id(sbml_dfs_table: pd.DataFrame) -> int:
+    """
+    Get Current Max ID
+
+    Look at a table from an SBML_dfs object and find the largest primary key following
+    the default naming convention for a the table.
+
+    Params:
+    sbml_dfs_table (pd.DataFrame):
+        A table derived from an SBML_dfs object.
+
+    Returns:
+    current_max_id (int):
+        The largest id which is already defined in the table using its expected naming
+        convention. If no IDs following this convention are present then the default
+        will be -1. In this way new IDs will be added starting with 0.
+
+    """
+
+    existing_ids_numeric = id_formatter_inv(sbml_dfs_table.index.tolist())
+
+    # filter np.nan which will be introduced if the key is not the default format
+    existing_ids_numeric_valid = [x for x in existing_ids_numeric if x is not np.nan]
+    if len(existing_ids_numeric_valid) == 0:
+        current_max_id = -1
+    else:
+        current_max_id = max(existing_ids_numeric_valid)
+
+    return current_max_id
+
+
+def adapt_pw_index(
+    source: str | indices.PWIndex,
+    species: str | Iterable[str] | None,
+    outdir: str | None = None,
+) -> indices.PWIndex:
+    """Adapts a pw_index
+
+    Helpful to filter for species before reconstructing.
+
+    Args:
+        source (str | PWIndex): uri for pw_index.csv file or PWIndex object
+        species (str):
+        outdir (str | None, optional): Optional directory to write pw_index to.
+            Defaults to None.
+
+    Returns:
+        indices.PWIndex: Filtered pw index
+    """
+    if isinstance(source, str):
+        pw_index = indices.PWIndex(source)
+    elif isinstance(source, indices.PWIndex):
+        pw_index = copy.deepcopy(source)
+    else:
+        raise ValueError("'source' needs to be str or PWIndex.")
+    pw_index.filter(species=species)
+
+    if outdir is not None:
+        with open_fs(outdir, create=True) as fs:
+            with fs.open("pw_index.tsv", "w") as f:
+                pw_index.index.to_csv(f, sep="\t")
+    return pw_index
+
+
+def _dogmatic_to_defining_bqbs(dogmatic: bool = False) -> str:
+    assert isinstance(dogmatic, bool)
+    if dogmatic:
+        logger.info(
+            "Running in dogmatic mode - differences genes, transcripts, and proteins will "
+            "try to be maintained as separate species."
+        )
+        # preserve differences between genes, transcripts, and proteins
+        defining_biological_qualifiers = BQB_DEFINING_ATTRS
+    else:
+        logger.info(
+            "Running in non-dogmatic mode - genes, transcripts, and proteins will "
+            "be merged if possible."
+        )
+        # merge genes, transcripts, and proteins (if they are defined with
+        # bqb terms which specify their relationships).
+        defining_biological_qualifiers = BQB_DEFINING_ATTRS_LOOSE
+
+    return defining_biological_qualifiers
+
+
+def match_entitydata_index_to_entity(
+    entity_data_dict: dict,
+    an_entity_data_type: str,
+    consensus_entity_df: pd.DataFrame,
+    entity_schema: dict,
+    table: str,
+) -> pd.DataFrame:
+    """
+    Match the index of entity_data_dict[an_entity_data_type] with the index of corresponding entity.
+    Update entity_data_dict[an_entity_data_type]'s index to the same as consensus_entity_df's index
+    Report cases where entity_data has indices not in corresponding entity's index.
+    Args
+        entity_data_dict (dict): dictionary containing all model's "an_entity_data_type" dictionaries
+        an_entity_data_type (str): data_type from species/reactions_data in entity_data_dict
+        consensus_entity_df (pd.DataFrame): the dataframe of the corresponding entity
+        entity_schema (dict): schema for "table"
+        table (str): table whose data is being consolidates (currently species or reactions)
+    Returns:
+        entity_data_df (pd.DataFrame) table for entity_data_dict[an_entity_data_type]
+    """
+
+    data_table = table + "_data"
+    entity_data_df = entity_data_dict[an_entity_data_type]
+
+    # ensure entity_data_df[an_entity_data_type]'s index doesn't have
+    # reaction ids that are not in consensus_entity's index
+    if len(entity_data_df.index.difference(consensus_entity_df.index)) == 0:
+        logger.info(f"{data_table} ids are included in {table} ids")
+    else:
+        logger.warnning(
+            f"{data_table} have ids are not matched to {table} ids,"
+            f"please check mismatched ids first"
+        )
+
+    # when entity_data_df is only a subset of the index of consensus_entity_df
+    # add ids only in consensus_entity_df to entity_data_df, and fill values with Nan
+    if len(entity_data_df) != len(consensus_entity_df):
+        logger.info(
+            f"The {data_table} has {len(entity_data_df)} ids,"
+            f"different from {len(consensus_entity_df)} ids in the {table} table,"
+            f"updating {data_table} ids."
+        )
+
+        entity_data_df = pd.concat(
+            [
+                entity_data_df,
+                consensus_entity_df[
+                    ~consensus_entity_df.index.isin(entity_data_df.index)
+                ],
+            ],
+            ignore_index=False,
+        )
+
+        entity_data_df.drop(entity_schema["vars"], axis=1, inplace=True)
+
+    return entity_data_df
+
+
+def check_entity_data_index_matching(sbml_dfs, table):
+    """
+    Update the input smbl_dfs's entity_data (dict) index
+    with match_entitydata_index_to_entity,
+    so that index for dataframe(s) in entity_data (dict) matches the sbml_dfs'
+    corresponding entity, and then passes sbml_dfs.validate()
+    Args
+        sbml_dfs (cpr.SBML_dfs): a cpr.SBML_dfs
+        table (str): table whose data is being consolidates (currently species or reactions)
+    Returns
+        sbml_dfs (cpr.SBML_dfs):
+        sbml_dfs whose entity_data is checked to have the same index
+        as the corresponding entity.
+    """
+
+    table_data = table + "_data"
+
+    entity_data_dict = getattr(sbml_dfs, table_data)
+    entity_schema = sbml_dfs.schema[table]
+    sbml_dfs_entity = getattr(sbml_dfs, table)
+
+    if entity_data_dict != {}:
+        entity_data_types = set.union(set(entity_data_dict.keys()))
+
+        entity_data_dict_checked = {
+            x: match_entitydata_index_to_entity(
+                entity_data_dict, x, sbml_dfs_entity, entity_schema, table
+            )
+            for x in entity_data_types
+        }
+
+        if table == SBML_DFS.REACTIONS:
+            sbml_dfs.reactions_data = entity_data_dict_checked
+        elif table == SBML_DFS.SPECIES:
+            sbml_dfs.species_data = entity_data_dict_checked
+
+    return sbml_dfs
+
+
+def _dogmatic_to_defining_bqbs(dogmatic: bool = False) -> str:
+    assert isinstance(dogmatic, bool)
+    if dogmatic:
+        logger.info(
+            "Running in dogmatic mode - differences genes, transcripts, and proteins will "
+            "try to be maintained as separate species."
+        )
+        # preserve differences between genes, transcripts, and proteins
+        defining_biological_qualifiers = BQB_DEFINING_ATTRS
+    else:
+        logger.info(
+            "Running in non-dogmatic mode - genes, transcripts, and proteins will "
+            "be merged if possible."
+        )
+        # merge genes, transcripts, and proteins (if they are defined with
+        # bqb terms which specify their relationships).
+        defining_biological_qualifiers = BQB_DEFINING_ATTRS_LOOSE
+
+    return defining_biological_qualifiers
+
+
+def _stub_ids(ids):
+    """Stub with a blank ID if an ids list is blank; otherwise create an Identifiers object from the provided ids"""
+    if len(ids) == 0:
+        return pd.DataFrame(
+            {
+                IDENTIFIERS.ONTOLOGY: [None],
+                IDENTIFIERS.IDENTIFIER: [None],
+                IDENTIFIERS.URL: [None],
+                IDENTIFIERS.BQB: [None],
+            }
+        )
+    else:
+        return pd.DataFrame(ids)

napistu/source.py

@@ -0,0 +1,394 @@
+from __future__ import annotations
+
+import pandas as pd
+from napistu import indices
+from napistu.constants import SOURCE_SPEC
+
+
+class Source:
+    """
+    An Entity's Source
+
+    Attributes
+    ----------
+    source : pd.DataFrame
+        A dataframe containing the model source and other optional variables
+
+    Methods
+    -------
+
+    """
+
+    def __init__(
+        self,
+        source_df: pd.DataFrame | None = None,
+        init: bool = False,
+        pw_index: indices.PWIndex | None = None,
+    ) -> None:
+        """
+        Tracks the model(s) an entity (i.e., a compartment, species, reaction) came from.
+
+        By convention sources exist only for the models that an entity came from rather
+        than the current model they are part of. For example, when combining Reactome models
+        into a consensus, a molecule which existed in multiple models would have a source entry
+        for each, but it would not have a source entry for the consensus model itself.
+
+        Parameters
+        ----------
+        source_df : pd.DataFrame
+            A dataframe containing the model source and other optional variables
+        init : bool
+            Creates an empty source object. This is typically used when creating an SBML_dfs
+            object from a single source.
+        pw_index : indices.PWIndex
+
+        Returns
+        -------
+        None.
+
+        """
+
+        if init is True:
+            # initialize with an empty Source
+            self.source = None
+        else:
+            if isinstance(source_df, pd.DataFrame):
+                # if pw_index is provided then it will be joined to source_df to add additional metadata
+                if pw_index is not None:
+                    if not isinstance(pw_index, indices.PWIndex):
+                        raise ValueError(
+                            f"pw_index must be a indices.PWIndex or None and was {type(pw_index).__name__}"
+                        )
+                    else:
+                        # check that all models are present in the pathway index
+                        missing_pathways = set(
+                            source_df[SOURCE_SPEC.MODEL].tolist()
+                        ).difference(
+                            set(pw_index.index[SOURCE_SPEC.PATHWAY_ID].tolist())
+                        )
+                        if len(missing_pathways) > 0:
+                            raise ValueError(
+                                f"{len(missing_pathways)} pathway models are present"
+                                f" in source_df but not the pw_index: {', '.join(missing_pathways)}"
+                            )
+
+                        source_df = source_df.merge(
+                            pw_index.index,
+                            left_on=SOURCE_SPEC.MODEL,
+                            right_on=SOURCE_SPEC.PATHWAY_ID,
+                        )
+
+                self.source = source_df
+            else:
+                raise TypeError(
+                    'source_df must be a pd.DataFrame if "init" is False, but was type '
+                    f"{type(source_df).__name__}"
+                )
+
+            if SOURCE_SPEC.MODEL not in source_df.columns.values.tolist():
+                raise ValueError(
+                    f"{SOURCE_SPEC.MODEL} variable was not found, but is required in a Source object"
+                )
+            if SOURCE_SPEC.PATHWAY_ID not in source_df.columns.values.tolist():
+                raise ValueError(
+                    f"{SOURCE_SPEC.PATHWAY_ID} variable was not found, but is required in a Source object"
+                )
+
+
+def create_source_table(
+    lookup_table: pd.Series, table_schema: dict, pw_index: indices.PWIndex | None
+) -> pd.DataFrame:
+    """
+    Create Source Table
+
+    Create a table with one row per "new_id" and a Source object created from the union
+      of "old_id" Source objects
+    """
+
+    if SOURCE_SPEC.SOURCE not in table_schema.keys():
+        raise ValueError(
+            f"{SOURCE_SPEC.SOURCE} not present in schema, can't create source_table"
+        )
+
+    # take lookup_table and create an index on "new_id". Multiple rows may have the
+    # same value for new_id so these are grouped together.
+    lookup_table_rearranged = lookup_table.reset_index().set_index(["new_id"])
+
+    # run a list comprehension over each value of new_id to create a Source
+    # object based on the dataframe specific to new_id
+    # pw_index is provided to fill out additional meta-information beyond the
+    # pathway_id which defines a single source
+    def create_source(group):
+        return Source(
+            group.reset_index(drop=True),
+            pw_index=pw_index,
+        )
+
+    id_table = (
+        lookup_table_rearranged.groupby("new_id")
+        .apply(create_source)
+        .rename(table_schema[SOURCE_SPEC.SOURCE])
+        .to_frame()
+    )
+
+    id_table.index = id_table.index.rename(table_schema["pk"])
+
+    return id_table
+
+
+def merge_sources(source_list: list | pd.Series) -> Source:
+    """
+    Merge Sources
+
+    Merge a list of Source objects into a single Source object
+
+    """
+
+    # filter to non-empty sources
+    # empty sources have only been initialized; a merge hasn't occured
+    existing_sources = [s.source is not None for s in source_list]
+    if not any(existing_sources):
+        if isinstance(source_list, list):
+            return source_list[0]
+        else:
+            return source_list.iloc[0]
+
+    existing_source_list = [
+        x.source for x, y in zip(source_list, existing_sources) if y
+    ]
+
+    return Source(pd.concat(existing_source_list))
+
+
+def unnest_sources(
+    source_table: pd.DataFrame, source_var: str, verbose: bool = False
+) -> pd.DataFrame:
+    """
+    Unnest Sources
+
+    Take a pd.DataFrame containing an array of Sources and
+    return one-row per source.
+
+    Parameters:
+    source_table: pd.DataFrame
+        a table containing an array of Sources
+    source_var: str
+        variable containing Sources
+
+    Returns:
+    pd.Dataframe containing the index of source_table but expanded
+    to include one row per source
+
+    """
+
+    sources = list()
+    source_table_index = source_table.index.to_frame().reset_index(drop=True)
+
+    for i in range(source_table.shape[0]):
+        if verbose:
+            print(f"Processing {source_table_index.index.values[i]}")
+
+        # check that the entries of sourcevar are Source objects
+        source_value = source_table[source_var].iloc[i]
+
+        assert isinstance(source_value, Source)
+
+        if source_value.source is None:
+            print("Some sources were only missing - returning None")
+            return None
+
+        source_tbl = pd.DataFrame(source_value.source)
+        source_tbl.index.name = SOURCE_SPEC.INDEX_NAME
+        source_tbl = source_tbl.reset_index()
+
+        # add original index as variables and then set index
+        for j in range(source_table_index.shape[1]):
+            source_tbl[source_table_index.columns[j]] = source_table_index.iloc[i, j]
+        source_tbl = source_tbl.set_index(
+            list(source_table_index.columns) + [SOURCE_SPEC.INDEX_NAME]
+        )
+
+        sources.append(source_tbl)
+
+    return pd.concat(sources)
+
+
+def greedy_set_coverge_of_sources(
+    source_df: pd.DataFrame, table_schema: dict
+) -> pd.DataFrame:
+    """
+    Greedy Set Coverage of Sources
+
+    Apply the greedy set coverge algorithm to find the minimal set of
+      sources which cover all entries
+
+    Parameters:
+    source_df: pd.DataFrame
+        pd.Dataframe containing the index of source_table but expanded to
+        include one row per source. As produced by source.unnest_sources()
+
+    Returns:
+    minimial_sources: [str]
+        A list of pathway_ids of the minimal source set
+
+    """
+
+    # rollup pathways with identical membership
+    deduplicated_sources = _deduplicate_source_df(source_df, table_schema)
+
+    unaccounted_for_members = deduplicated_sources
+    retained_pathway_ids = []
+
+    while unaccounted_for_members.shape[0] != 0:
+        # find the pathway with the most members
+        pathway_members = unaccounted_for_members.groupby(SOURCE_SPEC.PATHWAY_ID).size()
+        top_pathway = pathway_members[pathway_members == max(pathway_members)].index[0]
+        retained_pathway_ids.append(top_pathway)
+
+        # remove all members associated with the top pathway
+        members_captured = (
+            unaccounted_for_members[
+                unaccounted_for_members[SOURCE_SPEC.PATHWAY_ID] == top_pathway
+            ]
+            .index.get_level_values(table_schema["pk"])
+            .tolist()
+        )
+
+        unaccounted_for_members = unaccounted_for_members[
+            ~unaccounted_for_members.index.get_level_values(table_schema["pk"]).isin(
+                members_captured
+            )
+        ]
+
+    minimial_sources = deduplicated_sources[
+        deduplicated_sources[SOURCE_SPEC.PATHWAY_ID].isin(retained_pathway_ids)
+    ].sort_index()
+
+    return minimial_sources
+
+
+def _deduplicate_source_df(source_df: pd.DataFrame, table_schema: dict) -> pd.DataFrame:
+    """Combine entries in a source table when multiple models have the same members."""
+
+    # drop entries which are missing required attributes and throw an error if none are left
+    REQUIRED_NON_NA_ATTRIBUTES = [SOURCE_SPEC.PATHWAY_ID]
+    indexed_sources = (
+        source_df.reset_index()
+        .merge(source_df[REQUIRED_NON_NA_ATTRIBUTES].dropna())
+        .set_index(SOURCE_SPEC.PATHWAY_ID)
+    )
+
+    if indexed_sources.shape[0] == 0:
+        raise ValueError(
+            f"source_df was provided but zero entries had a defined {' OR '.join(REQUIRED_NON_NA_ATTRIBUTES)}"
+        )
+
+    pathways = indexed_sources.index.unique()
+
+    # identify pathways with identical coverage
+
+    pathway_member_string = (
+        pd.DataFrame(
+            [
+                {
+                    SOURCE_SPEC.PATHWAY_ID: p,
+                    "membership_string": "_".join(
+                        set(indexed_sources.loc[[p]][table_schema["pk"]].tolist())
+                    ),
+                }
+                for p in pathways
+            ]
+        )
+        .drop_duplicates()
+        .set_index("membership_string")
+    )
+
+    membership_categories = pathway_member_string.merge(
+        source_df.groupby(SOURCE_SPEC.PATHWAY_ID).first(),
+        left_on=SOURCE_SPEC.PATHWAY_ID,
+        right_index=True,
+    )
+
+    category_index = membership_categories.index.unique()
+    assert isinstance(category_index, pd.core.indexes.base.Index)
+
+    merged_sources = pd.concat(
+        [
+            _collapse_by_membership_string(s, membership_categories, table_schema)  # type: ignore
+            for s in category_index.tolist()
+        ]
+    )
+    merged_sources[SOURCE_SPEC.INDEX_NAME] = merged_sources.groupby(
+        table_schema["pk"]
+    ).cumcount()
+
+    return merged_sources.set_index(
+        [table_schema["pk"], SOURCE_SPEC.INDEX_NAME]
+    ).sort_index()
+
+
+def _collapse_by_membership_string(
+    membership_string: str, membership_categories: pd.DataFrame, table_schema: dict
+) -> pd.DataFrame:
+    """Assign each member of a membership-string to a set of pathways."""
+
+    collapsed_source_membership = _collapse_source_df(
+        membership_categories.loc[membership_string]
+    )
+
+    return pd.DataFrame(
+        [
+            pd.concat(
+                [pd.Series({table_schema["pk"]: ms}), collapsed_source_membership]
+            )
+            for ms in membership_string.split("_")
+        ]
+    )
+
+
+def _collapse_source_df(source_df: pd.DataFrame) -> pd.Series:
+    """Collapse a source_df table into a single entry."""
+
+    if isinstance(source_df, pd.DataFrame):
+        collapsed_source_series = pd.Series(
+            {
+                SOURCE_SPEC.PATHWAY_ID: " OR ".join(source_df[SOURCE_SPEC.PATHWAY_ID]),
+                SOURCE_SPEC.MODEL: " OR ".join(source_df[SOURCE_SPEC.MODEL]),
+                SOURCE_SPEC.SOURCE: " OR ".join(
+                    set(source_df[SOURCE_SPEC.SOURCE].tolist())
+                ),
+                SOURCE_SPEC.SPECIES: " OR ".join(
+                    set(source_df[SOURCE_SPEC.SPECIES].tolist())
+                ),
+                SOURCE_SPEC.NAME: " OR ".join(source_df[SOURCE_SPEC.NAME]),
+                SOURCE_SPEC.N_COLLAPSED_PATHWAYS: source_df.shape[0],
+            }
+        )
+    elif isinstance(source_df, pd.Series):
+        collapsed_source_series = pd.Series(
+            {
+                SOURCE_SPEC.PATHWAY_ID: source_df[SOURCE_SPEC.PATHWAY_ID],
+                SOURCE_SPEC.MODEL: source_df[SOURCE_SPEC.MODEL],
+                SOURCE_SPEC.SOURCE: source_df[SOURCE_SPEC.SOURCE],
+                SOURCE_SPEC.SPECIES: source_df[SOURCE_SPEC.SPECIES],
+                SOURCE_SPEC.NAME: source_df[SOURCE_SPEC.NAME],
+                SOURCE_SPEC.N_COLLAPSED_PATHWAYS: 1,
+            }
+        )
+    else:
+        raise TypeError(
+            f"source_df must be a pd.DataFrame or pd.Series, but was a {type(source_df).__name__}"
+        )
+
+    return collapsed_source_series
+
+
+def _safe_source_merge(member_Sources: Source | list) -> Source:
+    """Combine either a Source or pd.Series of Sources into a single Source object."""
+
+    if isinstance(member_Sources, Source):
+        return member_Sources
+    elif isinstance(member_Sources, pd.Series):
+        return merge_sources(member_Sources.tolist())
+    else:
+        raise TypeError("Expecting source.Source or pd.Series")