napistu 0.3.6__py3-none-any.whl → 0.4.0__py3-none-any.whl

napistu/sbml_dfs_core.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import logging
 import re
 from typing import Any
@@ -64,6 +65,8 @@ class SBML_dfs:
         Add a new reactions data table to the model with validation.
     add_species_data(label, data)
         Add a new species data table to the model with validation.
+    copy()
+        Return a deep copy of the SBML_dfs object.
     export_sbml_dfs(model_prefix, outdir, overwrite=False, dogmatic=True)
         Export the SBML_dfs model and its tables to files in a specified directory.
     get_characteristic_species_ids(dogmatic=True)
@@ -114,7 +117,6 @@ class SBML_dfs:
     Private/Hidden Methods (alphabetical, appear after public methods)
     -----------------------------------------------------------------
     _attempt_resolve(e)
-    _check_pk_fk_correspondence()
     _find_underspecified_reactions_by_scids(sc_ids)
     _get_unused_cspecies()
     _get_unused_species()
@@ -123,9 +125,12 @@ class SBML_dfs:
     _remove_species(s_ids)
     _remove_unused_cspecies()
     _remove_unused_species()
+    _validate_identifiers()
+    _validate_pk_fk_correspondence()
     _validate_r_ids(r_ids)
     _validate_reaction_species()
     _validate_reactions_data(reactions_data_table)
+    _validate_sources()
     _validate_species_data(species_data_table)
     _validate_table(table_name)
     """
@@ -255,6 +260,17 @@ class SBML_dfs:
             )
         self.species_data[label] = data
 
+    def copy(self):
+        """
+        Return a deep copy of the SBML_dfs object.
+
+        Returns
+        -------
+        SBML_dfs
+            A deep copy of the current SBML_dfs object.
+        """
+        return copy.deepcopy(self)
+
     def export_sbml_dfs(
         self,
         model_prefix: str,
@@ -440,7 +456,7 @@ class SBML_dfs:
             If id_type is invalid or identifiers are malformed
         """
         selected_table = self.get_table(id_type, {"id"})
-        schema = self.schema
+        schema = SBML_DFS_SCHEMA.SCHEMA
 
         identifiers_dict = dict()
         for sysid in selected_table.index:
@@ -458,6 +474,7 @@ class SBML_dfs:
         if not identifiers_dict:
             # Return empty DataFrame with expected columns if nothing found
             return pd.DataFrame(columns=[schema[id_type]["pk"], "entry"])
+
         identifiers_tbl = pd.concat(identifiers_dict)
 
         identifiers_tbl.index.names = [schema[id_type]["pk"], "entry"]
@@ -1382,7 +1399,7 @@ class SBML_dfs:
             self._validate_table(table)
 
         # check whether pks and fks agree
-        self._check_pk_fk_correspondence()
+        self._validate_pk_fk_correspondence()
 
         # check optional data tables:
         for k, v in self.species_data.items():
@@ -1400,6 +1417,10 @@ class SBML_dfs:
         # validate reaction_species sbo_terms and stoi
         self._validate_reaction_species()
 
+        # validate identifiers and sources
+        self._validate_identifiers()
+        self._validate_sources()
+
     def validate_and_resolve(self):
         """
         Validate and attempt to automatically fix common issues.
@@ -1455,67 +1476,6 @@ class SBML_dfs:
             )
             raise e
 
-    def _check_pk_fk_correspondence(self):
-        """
-        Check whether primary keys and foreign keys agree for all tables in the schema.
-        Raises ValueError if any correspondence fails.
-        """
-
-        pk_df = pd.DataFrame(
-            [{"pk_table": k, "key": v["pk"]} for k, v in self.schema.items()]
-        )
-
-        fk_df = (
-            pd.DataFrame(
-                [
-                    {"fk_table": k, "fk": v["fk"]}
-                    for k, v in self.schema.items()
-                    if "fk" in v.keys()
-                ]
-            )
-            .set_index("fk_table")["fk"]
-            .apply(pd.Series)
-            .reset_index()
-            .melt(id_vars="fk_table")
-            .drop(["variable"], axis=1)
-            .rename(columns={"value": "key"})
-        )
-
-        pk_fk_correspondences = pk_df.merge(fk_df)
-
-        for i in range(0, pk_fk_correspondences.shape[0]):
-            pk_table_keys = set(
-                getattr(self, pk_fk_correspondences["pk_table"][i]).index.tolist()
-            )
-            if None in pk_table_keys:
-                raise ValueError(
-                    f"{pk_fk_correspondences['pk_table'][i]} had "
-                    "missing values in its index"
-                )
-
-            fk_table_keys = set(
-                getattr(self, pk_fk_correspondences["fk_table"][i]).loc[
-                    :, pk_fk_correspondences["key"][i]
-                ]
-            )
-            if None in fk_table_keys:
-                raise ValueError(
-                    f"{pk_fk_correspondences['fk_table'][i]} included "
-                    f"missing {pk_fk_correspondences['key'][i]} values"
-                )
-
-            # all foreign keys need to match a primary key
-            extra_fks = fk_table_keys.difference(pk_table_keys)
-            if len(extra_fks) != 0:
-                raise ValueError(
-                    f"{len(extra_fks)} distinct "
-                    f"{pk_fk_correspondences['key'][i]} values were"
-                    f" found in {pk_fk_correspondences['fk_table'][i]} "
-                    f"but missing from {pk_fk_correspondences['pk_table'][i]}."
-                    " All foreign keys must have a matching primary key.\n\n"
-                    f"Extra key are: {', '.join(extra_fks)}"
-                )
-
     def _find_underspecified_reactions_by_scids(
         self, sc_ids: Iterable[str]
     ) -> set[str]:
@@ -1640,6 +1600,88 @@ class SBML_dfs:
         s_ids = self._get_unused_species()
         self._remove_species(s_ids)
 
+    def _validate_identifiers(self):
+        """
+        Validate identifiers in the model
+
+        Iterates through all tables and checks if the identifier columns are valid.
+
+        Raises:
+            ValueError: missing identifiers in the table
+        """
+
+        SCHEMA = SBML_DFS_SCHEMA.SCHEMA
+        for table in SBML_DFS_SCHEMA.SCHEMA.keys():
+            if "id" not in SCHEMA[table].keys():
+                continue
+            id_series = self.get_table(table)[SCHEMA[table]["id"]]
+            if id_series.isna().sum() > 0:
+                missing_ids = id_series[id_series.isna()].index
+                raise ValueError(
+                    f"{table} has {len(missing_ids)} missing ids: {missing_ids}"
+                )
+
+    def _validate_pk_fk_correspondence(self):
+        """
+        Check whether primary keys and foreign keys agree for all tables in the schema.
+        Raises ValueError if any correspondence fails.
+        """
+
+        pk_df = pd.DataFrame(
+            [{"pk_table": k, "key": v["pk"]} for k, v in self.schema.items()]
+        )
+
+        fk_df = (
+            pd.DataFrame(
+                [
+                    {"fk_table": k, "fk": v["fk"]}
+                    for k, v in self.schema.items()
+                    if "fk" in v.keys()
+                ]
+            )
+            .set_index("fk_table")["fk"]
+            .apply(pd.Series)
+            .reset_index()
+            .melt(id_vars="fk_table")
+            .drop(["variable"], axis=1)
+            .rename(columns={"value": "key"})
+        )
+
+        pk_fk_correspondences = pk_df.merge(fk_df)
+
+        for i in range(0, pk_fk_correspondences.shape[0]):
+            pk_table_keys = set(
+                getattr(self, pk_fk_correspondences["pk_table"][i]).index.tolist()
+            )
+            if None in pk_table_keys:
+                raise ValueError(
+                    f"{pk_fk_correspondences['pk_table'][i]} had "
+                    "missing values in its index"
+                )
+
+            fk_table_keys = set(
+                getattr(self, pk_fk_correspondences["fk_table"][i]).loc[
+                    :, pk_fk_correspondences["key"][i]
+                ]
+            )
+            if None in fk_table_keys:
+                raise ValueError(
+                    f"{pk_fk_correspondences['fk_table'][i]} included "
+                    f"missing {pk_fk_correspondences['key'][i]} values"
+                )
+
+            # all foreign keys need to match a primary key
+            extra_fks = fk_table_keys.difference(pk_table_keys)
+            if len(extra_fks) != 0:
+                raise ValueError(
+                    f"{len(extra_fks)} distinct "
+                    f"{pk_fk_correspondences['key'][i]} values were"
+                    f" found in {pk_fk_correspondences['fk_table'][i]} "
+                    f"but missing from {pk_fk_correspondences['pk_table'][i]}."
+                    " All foreign keys must have a matching primary key.\n\n"
+                    f"Extra key are: {', '.join(extra_fks)}"
+                )
+
     def _validate_r_ids(self, r_ids: Optional[Union[str, list[str]]]) -> list[str]:
 
         if isinstance(r_ids, str):
@@ -1694,6 +1736,27 @@ class SBML_dfs:
         """
         sbml_dfs_utils._validate_matching_data(reactions_data_table, self.reactions)
 
+    def _validate_sources(self):
+        """
+        Validate sources in the model
+
+        Iterates through all tables and checks if the source columns are valid.
+
+        Raises:
+            ValueError: missing sources in the table
+        """
+
+        SCHEMA = SBML_DFS_SCHEMA.SCHEMA
+        for table in SBML_DFS_SCHEMA.SCHEMA.keys():
+            if "source" not in SCHEMA[table].keys():
+                continue
+            source_series = self.get_table(table)[SCHEMA[table]["source"]]
+            if source_series.isna().sum() > 0:
+                missing_sources = source_series[source_series.isna()].index
+                raise ValueError(
+                    f"{table} has {len(missing_sources)} missing sources: {missing_sources}"
+                )
+
     def _validate_species_data(self, species_data_table: pd.DataFrame):
         """Validates species data attribute
 

napistu/sbml_dfs_utils.py

@@ -27,6 +27,8 @@ from napistu.constants import MINI_SBO_FROM_NAME
 from napistu.constants import MINI_SBO_TO_NAME
 from napistu.constants import SBO_NAME_TO_ROLE
 from napistu.constants import ONTOLOGIES
+from napistu.constants import VALID_SBO_TERM_NAMES
+from napistu.constants import VALID_SBO_TERMS
 from napistu.ingestion.constants import VALID_COMPARTMENTS
 from napistu.ingestion.constants import COMPARTMENTS_GO_TERMS
 from napistu.ingestion.constants import GENERIC_COMPARTMENT
@@ -559,6 +561,10 @@ def unnest_identifiers(id_table: pd.DataFrame, id_var: str) -> pd.DataFrame:
 
     N_invalid_ids = sum(id_table[id_var].isna())
     if N_invalid_ids != 0:
+
+        print("Rows with missing identifiers:")
+        print(id_table.loc[id_table[id_var].isna(), id_var])
+
         raise ValueError(
             f'{N_invalid_ids} entries in "id_table" were missing',
             "entries with no identifiers should still include an Identifiers object",
@@ -1277,3 +1283,47 @@ def _validate_matching_data(data_table: pd.DataFrame, ref_table: pd.DataFrame):
             f"The data table was type {type(data_table).__name__}"
             " but must be a pd.DataFrame"
         )
+
+
+def _validate_sbo_values(sbo_series: pd.Series, validate: str = "names") -> None:
+    """
+    Validate SBO terms or names
+
+    Parameters
+    ----------
+    sbo_series : pd.Series
+        The SBO terms or names to validate.
+    validate : str, optional
+        Whether the values are SBO terms ("terms") or names ("names", default).
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    ValueError
+        If the validation type is invalid.
+    TypeError
+        If the invalid_counts is not a pandas DataFrame.
+    ValueError
+        If some reaction species have unusable SBO terms.
+    """
+
+    if validate == "terms":
+        valid_values = VALID_SBO_TERMS
+    elif validate == "names":
+        valid_values = VALID_SBO_TERM_NAMES
+    else:
+        raise ValueError(f"Invalid validation type: {validate}")
+
+    invalid_sbo_terms = sbo_series[~sbo_series.isin(valid_values)]
+
+    if invalid_sbo_terms.shape[0] != 0:
+        invalid_counts = invalid_sbo_terms.value_counts(sbo_series.name).to_frame("N")
+        if not isinstance(invalid_counts, pd.DataFrame):
+            raise TypeError("invalid_counts must be a pandas DataFrame")
+        print(invalid_counts)
+        raise ValueError("Some reaction species have unusable SBO terms")
+
+    return None

napistu/utils.py

@@ -7,24 +7,24 @@ import logging
 import os
 import pickle
 import re
+import requests
 import shutil
 import urllib.request as request
 import zipfile
 from contextlib import closing
 from itertools import starmap
 from textwrap import fill
-from typing import Any
-from typing import Union
-from typing import Optional
-from typing import List
+from typing import Any, List, Optional, Union
 from urllib.parse import urlparse
-import requests
+from pathlib import Path
 from requests.adapters import HTTPAdapter
 from requests.adapters import Retry
 
 import igraph as ig
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 from fs import open_fs
 from fs.copy import copy_dir
 from fs.copy import copy_file
@@ -604,6 +604,81 @@ def load_json(uri: str) -> Any:
         return json.loads(txt)
 
 
+def save_parquet(
+    df: pd.DataFrame, uri: Union[str, Path], compression: str = "snappy"
+) -> None:
+    """
+    Write a DataFrame to a single Parquet file.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        The DataFrame to save
+    uri : Union[str, Path]
+        Path where to save the Parquet file. Can be a local path or a GCS URI.
+        Recommended extensions: .parquet or .pq
+    compression : str, default 'snappy'
+        Compression algorithm. Options: 'snappy', 'gzip', 'brotli', 'lz4', 'zstd'
+
+    Raises
+    ------
+    OSError
+        If the file cannot be written to (permission issues, etc.)
+    """
+
+    uri_str = str(uri)
+
+    # Warn about non-standard extensions
+    if not any(uri_str.endswith(ext) for ext in [".parquet", ".pq"]):
+        logger.warning(
+            f"File '{uri_str}' doesn't have a standard Parquet extension (.parquet or .pq)"
+        )
+
+    target_base, target_path = get_target_base_and_path(uri_str)
+
+    with open_fs(target_base, create=True) as target_fs:
+        with target_fs.openbin(target_path, "w") as f:
+            # Convert to Arrow table and write as single file
+            table = pa.Table.from_pandas(df)
+            pq.write_table(
+                table,
+                f,
+                compression=compression,
+                use_dictionary=True,  # Efficient for repeated values
+                write_statistics=True,  # Enables query optimization
+            )
+
+
+def load_parquet(uri: Union[str, Path]) -> pd.DataFrame:
+    """
+    Read a DataFrame from a Parquet file.
+
+    Parameters
+    ----------
+    uri : Union[str, Path]
+        Path to the Parquet file to load
+
+    Returns
+    -------
+    pd.DataFrame
+        The DataFrame loaded from the Parquet file
+
+    Raises
+    ------
+    FileNotFoundError
+        If the specified file does not exist
+    """
+    try:
+        target_base, target_path = get_target_base_and_path(str(uri))
+
+        with open_fs(target_base) as target_fs:
+            with target_fs.openbin(target_path, "r") as f:
+                return pd.read_parquet(f, engine="pyarrow")
+
+    except ResourceNotFound as e:
+        raise FileNotFoundError(f"File not found: {uri}") from e
+
+
 def extract_regex_search(regex: str, query: str, index_value: int = 0) -> str:
     """
     Match an identifier substring and otherwise throw an error
@@ -810,50 +885,15 @@ def drop_extra_cols(
     return df_out.loc[:, ordered_cols]
 
 
-def _merge_and_log_overwrites(
-    left_df: pd.DataFrame, right_df: pd.DataFrame, merge_context: str, **merge_kwargs
-) -> pd.DataFrame:
+def update_pathological_names(names: pd.Series, prefix: str) -> pd.Series:
     """
-    Merge two DataFrames and log any column overwrites.
-
-    Parameters
-    ----------
-    left_df : pd.DataFrame
-        Left DataFrame for merge
-    right_df : pd.DataFrame
-        Right DataFrame for merge
-    merge_context : str
-        Description of the merge operation for logging
-    **merge_kwargs : dict
-        Additional keyword arguments passed to pd.merge
+    Update pathological names in a pandas Series.
 
-    Returns
-    -------
-    pd.DataFrame
-        Merged DataFrame with overwritten columns removed
+    Add a prefix to the names if they are all numeric.
     """
-    # Track original columns
-    original_cols = left_df.columns.tolist()
-
-    # Ensure we're using the correct suffixes
-    merge_kwargs["suffixes"] = ("_old", "")
-
-    # Perform merge
-    merged_df = pd.merge(left_df, right_df, **merge_kwargs)
-
-    # Check for and log any overwritten columns
-    new_cols = merged_df.columns.tolist()
-    overwritten_cols = [col for col in original_cols if col + "_old" in new_cols]
-    if overwritten_cols:
-        logger.warning(
-            f"The following columns were overwritten during {merge_context} merge and their original values "
-            f"have been suffixed with '_old': {', '.join(overwritten_cols)}"
-        )
-        # Drop the old columns
-        cols_to_drop = [col + "_old" for col in overwritten_cols]
-        merged_df = merged_df.drop(columns=cols_to_drop)
-
-    return merged_df
+    if names.apply(lambda x: x.isdigit()).all():
+        names = names.apply(lambda x: f"{prefix}{x}")
+    return names
 
 
 def format_identifiers_as_edgelist(
@@ -1108,3 +1148,49 @@ def _add_nameness_score(df, name_var):
 
     df.loc[:, "nameness_score"] = df[name_var].apply(score_nameness)
     return df
+
+
+def _merge_and_log_overwrites(
+    left_df: pd.DataFrame, right_df: pd.DataFrame, merge_context: str, **merge_kwargs
+) -> pd.DataFrame:
+    """
+    Merge two DataFrames and log any column overwrites.
+
+    Parameters
+    ----------
+    left_df : pd.DataFrame
+        Left DataFrame for merge
+    right_df : pd.DataFrame
+        Right DataFrame for merge
+    merge_context : str
+        Description of the merge operation for logging
+    **merge_kwargs : dict
+        Additional keyword arguments passed to pd.merge
+
+    Returns
+    -------
+    pd.DataFrame
+        Merged DataFrame with overwritten columns removed
+    """
+    # Track original columns
+    original_cols = left_df.columns.tolist()
+
+    # Ensure we're using the correct suffixes
+    merge_kwargs["suffixes"] = ("_old", "")
+
+    # Perform merge
+    merged_df = pd.merge(left_df, right_df, **merge_kwargs)
+
+    # Check for and log any overwritten columns
+    new_cols = merged_df.columns.tolist()
+    overwritten_cols = [col for col in original_cols if col + "_old" in new_cols]
+    if overwritten_cols:
+        logger.warning(
+            f"The following columns were overwritten during {merge_context} merge and their original values "
+            f"have been suffixed with '_old': {', '.join(overwritten_cols)}"
+        )
+        # Drop the old columns
+        cols_to_drop = [col + "_old" for col in overwritten_cols]
+        merged_df = merged_df.drop(columns=cols_to_drop)
+
+    return merged_df

{napistu-0.3.6.dist-info → napistu-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: napistu
-Version: 0.3.6
+Version: 0.4.0
 Summary: Connecting high-dimensional data to curated pathways
 Home-page: https://github.com/napistu/napistu-py
 Author: Sean Hackett
@@ -27,6 +27,7 @@ Requires-Dist: mygene<4.0.0,>=3.0.0
 Requires-Dist: numpy<3.0.0,>=1.24.0
 Requires-Dist: pandas<3.0.0,>=1.5.0
 Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: pyarrow<20.0.0,>=15.0.0
 Requires-Dist: python-libsbml
 Requires-Dist: requests>=2.25.0
 Requires-Dist: scipy<2.0.0,>=1.10.0
@@ -51,7 +52,6 @@ Requires-Dist: markdown>=3.4.0; extra == "mcp"
 Requires-Dist: jupyter-client>=7.0.0; extra == "mcp"
 Requires-Dist: nbformat>=5.0.0; extra == "mcp"
 Provides-Extra: rpy2
-Requires-Dist: pyarrow<19.0.0,>=15.0.0; extra == "rpy2"
 Requires-Dist: rpy2<4.0.0,>=3.5.0; extra == "rpy2"
 Requires-Dist: rpy2-arrow<1.0.0,>=0.1.0; extra == "rpy2"
 Provides-Extra: scverse